From 7021fb48410385d05ebf302b90b0a95257dd7e5f Mon Sep 17 00:00:00 2001 From: Bulat Date: Fri, 23 May 2025 14:22:43 +0000 Subject: [PATCH 01/16] [C++ SDK] Supported topic-to-table transactions in Query Service (#15984) --- examples/basic_example/basic_example.cpp | 8 +- .../topic_reader/transaction/application.cpp | 36 +- .../topic_reader/transaction/application.h | 10 +- examples/topic_writer/transaction/main.cpp | 33 +- include/ydb-cpp-sdk/client/query/client.h | 70 +- include/ydb-cpp-sdk/client/query/fwd.h | 2 +- include/ydb-cpp-sdk/client/query/query.h | 6 + include/ydb-cpp-sdk/client/query/tx.h | 33 - include/ydb-cpp-sdk/client/table/table.h | 11 +- .../ydb-cpp-sdk/client/topic/read_session.h | 7 +- .../ydb-cpp-sdk/client/topic/write_session.h | 17 +- include/ydb-cpp-sdk/client/types/tx/tx.h | 34 + .../impl/federated_write_session.h | 4 +- src/client/query/client.cpp | 162 +- src/client/query/impl/exec_query.cpp | 181 +- src/client/table/impl/table_client.h | 36 +- src/client/table/impl/transaction.cpp | 83 +- src/client/table/impl/transaction.h | 12 +- src/client/table/table.cpp | 23 +- src/client/topic/impl/read_session_impl.h | 6 +- src/client/topic/impl/read_session_impl.ipp | 6 +- src/client/topic/impl/transaction.cpp | 5 +- src/client/topic/impl/transaction.h | 10 +- src/client/topic/impl/write_session.cpp | 6 +- src/client/topic/impl/write_session.h | 6 +- src/client/topic/impl/write_session_impl.cpp | 6 +- src/client/topic/impl/write_session_impl.h | 2 +- src/client/topic/ut/topic_to_table_ut.cpp | 1922 +++++++++++++---- tests/integration/server_restart/main.cpp | 2 +- 29 files changed, 2006 insertions(+), 733 deletions(-) create mode 100644 include/ydb-cpp-sdk/client/types/tx/tx.h diff --git a/examples/basic_example/basic_example.cpp b/examples/basic_example/basic_example.cpp index 126f9eecd1..cc5839b047 100644 --- a/examples/basic_example/basic_example.cpp +++ b/examples/basic_example/basic_example.cpp @@ -286,8 +286,8 @@ void MultiStep(TQueryClient client) { } // Get the active transaction id - auto txId = resultValue.GetTransaction()->GetId(); - + auto tx = *resultValue.GetTransaction(); + // Processing the request result TResultSetParser parser(resultValue.GetResultSet(0)); parser.TryNextRow(); @@ -328,7 +328,7 @@ void MultiStep(TQueryClient client) { // and commit it at the end of the second query execution. auto result2 = session.ExecuteQuery( query2, - TTxControl::Tx(txId).CommitTx(), + TTxControl::Tx(tx).CommitTx(), params2).GetValueSync(); if (!result2.IsSuccess()) { @@ -381,7 +381,7 @@ void ExplicitTcl(TQueryClient client) { // Execute query. // Transaction control settings continues active transaction (tx) auto updateResult = session.ExecuteQuery(query, - TTxControl::Tx(tx.GetId()), + TTxControl::Tx(tx), params).GetValueSync(); if (!updateResult.IsSuccess()) { diff --git a/examples/topic_reader/transaction/application.cpp b/examples/topic_reader/transaction/application.cpp index d30891d392..70975fa19d 100644 --- a/examples/topic_reader/transaction/application.cpp +++ b/examples/topic_reader/transaction/application.cpp @@ -20,10 +20,10 @@ TApplication::TApplication(const TOptions& options) Driver.emplace(config); TopicClient.emplace(*Driver); - TableClient.emplace(*Driver); + QueryClient.emplace(*Driver); CreateTopicReadSession(options); - CreateTableSession(); + CreateQuerySession(); TablePath = options.TablePath; } @@ -40,15 +40,15 @@ void TApplication::CreateTopicReadSession(const TOptions& options) std::cout << "Topic session was created" << std::endl; } -void TApplication::CreateTableSession() +void TApplication::CreateQuerySession() { - NYdb::NTable::TCreateSessionSettings settings; + NYdb::NQuery::TCreateSessionSettings settings; - auto result = TableClient->GetSession(settings).GetValueSync(); + auto result = QueryClient->GetSession(settings).GetValueSync(); - TableSession = result.GetSession(); + QuerySession = result.GetSession(); - std::cout << "Table session was created" << std::endl; + std::cout << "Query session was created" << std::endl; } void TApplication::Run() @@ -104,10 +104,10 @@ void TApplication::Finalize() void TApplication::BeginTransaction() { Y_ABORT_UNLESS(!Transaction); - Y_ABORT_UNLESS(TableSession); + Y_ABORT_UNLESS(QuerySession); - auto settings = NYdb::NTable::TTxSettings::SerializableRW(); - auto result = TableSession->BeginTransaction(settings).GetValueSync(); + auto settings = NYdb::NQuery::TTxSettings::SerializableRW(); + auto result = QuerySession->BeginTransaction(settings).GetValueSync(); Transaction = result.GetTransaction(); } @@ -116,7 +116,7 @@ void TApplication::CommitTransaction() { Y_ABORT_UNLESS(Transaction); - NYdb::NTable::TCommitTxSettings settings; + NYdb::NQuery::TCommitTxSettings settings; auto result = Transaction->Commit(settings).GetValueSync(); @@ -173,20 +173,16 @@ void TApplication::InsertRowsIntoTable() auto params = builder.Build(); - NYdb::NTable::TExecDataQuerySettings settings; - settings.KeepInQueryCache(true); - - auto runQuery = [this, &query, ¶ms, &settings](NYdb::NTable::TSession) -> NYdb::TStatus { + auto runQuery = [this, &query, ¶ms](NYdb::NQuery::TSession) -> NYdb::TStatus { auto result = - Transaction->GetSession().ExecuteDataQuery(query, - NYdb::NTable::TTxControl::Tx(*Transaction), - params, - settings).GetValueSync(); + Transaction->GetSession().ExecuteQuery(query, + NYdb::NQuery::TTxControl::Tx(*Transaction), + params).GetValueSync(); return result; }; - TableClient->RetryOperationSync(runQuery); + QueryClient->RetryQuerySync(runQuery); } void TApplication::AppendTableRow(const NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent::TMessage& message) diff --git a/examples/topic_reader/transaction/application.h b/examples/topic_reader/transaction/application.h index 9b81327c72..446dae3929 100644 --- a/examples/topic_reader/transaction/application.h +++ b/examples/topic_reader/transaction/application.h @@ -4,7 +4,7 @@ #include #include -#include +#include #include #include @@ -28,7 +28,7 @@ class TApplication { }; void CreateTopicReadSession(const TOptions& options); - void CreateTableSession(); + void CreateQuerySession(); void BeginTransaction(); void CommitTransaction(); @@ -40,10 +40,10 @@ class TApplication { std::optional Driver; std::optional TopicClient; - std::optional TableClient; + std::optional QueryClient; std::shared_ptr ReadSession; - std::optional TableSession; - std::optional Transaction; + std::optional QuerySession; + std::optional Transaction; std::vector PendingStopEvents; std::vector Rows; std::string TablePath; diff --git a/examples/topic_writer/transaction/main.cpp b/examples/topic_writer/transaction/main.cpp index c98c8df8e1..cd1719f8c0 100644 --- a/examples/topic_writer/transaction/main.cpp +++ b/examples/topic_writer/transaction/main.cpp @@ -1,8 +1,7 @@ #include -#include +#include -int main() -{ +int main() { const std::string ENDPOINT = "HOST:PORT"; const std::string DATABASE = "DATABASE"; const std::string TOPIC = "PATH/TO/TOPIC"; @@ -12,24 +11,26 @@ int main() config.SetDatabase(DATABASE); NYdb::TDriver driver(config); - NYdb::NTable::TTableClient tableClient(driver); - auto getTableSessionResult = tableClient.GetSession().GetValueSync(); - ThrowOnError(getTableSessionResult); - auto tableSession = getTableSessionResult.GetSession(); + NYdb::NQuery::TQueryClient queryClient(driver); + auto getSessionResult = queryClient.GetSession().GetValueSync(); + NYdb::NStatusHelpers::ThrowOnError(getSessionResult); + auto session = getSessionResult.GetSession(); NYdb::NTopic::TTopicClient topicClient(driver); - auto topicSessionSettings = NYdb::NTopic::TWriteSessionSettings() - .Path(TOPIC) - .DeduplicationEnabled(true); - auto topicSession = topicClient.CreateSimpleBlockingWriteSession(topicSessionSettings); - auto beginTransactionResult = tableSession.BeginTransaction().GetValueSync(); - ThrowOnError(beginTransactionResult); - auto transaction = beginTransactionResult.GetTransaction(); + auto topicSession = topicClient.CreateSimpleBlockingWriteSession( + NYdb::NTopic::TWriteSessionSettings() + .Path(TOPIC) + .DeduplicationEnabled(true) + ); + + auto beginTxResult = session.BeginTransaction(NYdb::NQuery::TTxSettings()).GetValueSync(); + NYdb::NStatusHelpers::ThrowOnError(beginTxResult); + auto tx = beginTxResult.GetTransaction(); NYdb::NTopic::TWriteMessage writeMessage("message"); - topicSession->Write(std::move(writeMessage), &transaction); + topicSession->Write(std::move(writeMessage), &tx); - transaction.Commit().GetValueSync(); + tx.Commit().GetValueSync(); } diff --git a/include/ydb-cpp-sdk/client/query/client.h b/include/ydb-cpp-sdk/client/query/client.h index eee15547c1..0764d52eb5 100644 --- a/include/ydb-cpp-sdk/client/query/client.h +++ b/include/ydb-cpp-sdk/client/query/client.h @@ -8,6 +8,7 @@ #include #include #include +#include #include namespace NYdb::inline V3 { @@ -168,30 +169,75 @@ class TCreateSessionResult: public TStatus { TSession Session_; }; -class TTransaction { +class TTransaction : public TTransactionBase { friend class TQueryClient; friend class TExecuteQueryIterator::TReaderImpl; + friend class TExecQueryImpl; + public: - const std::string& GetId() const { - return TxId_; + bool IsActive() const; + + TAsyncCommitTransactionResult Commit(const TCommitTxSettings& settings = TCommitTxSettings()); + TAsyncStatus Rollback(const TRollbackTxSettings& settings = TRollbackTxSettings()); + + TSession GetSession() const; + + void AddPrecommitCallback(TPrecommitTransactionCallback cb) override; + void AddOnFailureCallback(TOnFailureTransactionCallback cb) override; + +private: + TTransaction(const TSession& session, const std::string& txId); + + TAsyncStatus Precommit() const; + NThreading::TFuture ProcessFailure() const; + + class TImpl; + + std::shared_ptr TransactionImpl_; +}; + +class TTxControl { + friend class TExecQueryImpl; + friend class TExecQueryInternal; + +public: + using TSelf = TTxControl; + + static TTxControl Tx(const TTransaction& tx) { + return TTxControl(tx); } - bool IsActive() const { - return !TxId_.empty(); + [[deprecated("This is bug-provoking API. Use TTxControl::Tx(TTransaction) instead. " + "This constructor will be removed in upcomming release")]] + static TTxControl Tx(const std::string& txId) { + return TTxControl(txId); } - TAsyncCommitTransactionResult Commit(const TCommitTxSettings& settings = TCommitTxSettings()); - TAsyncStatus Rollback(const TRollbackTxSettings& settings = TRollbackTxSettings()); + static TTxControl BeginTx(const TTxSettings& settings = TTxSettings()) { + return TTxControl(settings); + } - TSession GetSession() const { - return Session_; + static TTxControl NoTx() { + return TTxControl(); } + FLUENT_SETTING_FLAG(CommitTx); + + bool HasTx() const { return !std::holds_alternative(Tx_); } + private: - TTransaction(const TSession& session, const std::string& txId); + TTxControl() {} - TSession Session_; - std::string TxId_; + TTxControl(const TTransaction& tx) + : Tx_(tx) {} + + TTxControl(const TTxSettings& txSettings) + : Tx_(txSettings) {} + + TTxControl(const std::string& txId) + : Tx_(txId) {} + + const std::variant Tx_; }; class TBeginTransactionResult : public TStatus { diff --git a/include/ydb-cpp-sdk/client/query/fwd.h b/include/ydb-cpp-sdk/client/query/fwd.h index 08b7f4e675..9f88fdc3c0 100644 --- a/include/ydb-cpp-sdk/client/query/fwd.h +++ b/include/ydb-cpp-sdk/client/query/fwd.h @@ -27,7 +27,7 @@ class TExecuteQueryPart; class TExecuteQueryIterator; class TTransaction; -struct TTxControl; +class TTxControl; class TQueryContent; class TResultSetMeta; diff --git a/include/ydb-cpp-sdk/client/query/query.h b/include/ydb-cpp-sdk/client/query/query.h index 7e2f1788fa..a73501d623 100644 --- a/include/ydb-cpp-sdk/client/query/query.h +++ b/include/ydb-cpp-sdk/client/query/query.h @@ -64,6 +64,12 @@ class TExecuteQueryIterator : public TStatus { : TStatus(std::move(status)) , ReaderImpl_(impl) {} + TExecuteQueryIterator( + std::shared_ptr impl, + TStatus&& status) + : TStatus(std::move(status)) + , ReaderImpl_(impl) {} + std::shared_ptr ReaderImpl_; }; diff --git a/include/ydb-cpp-sdk/client/query/tx.h b/include/ydb-cpp-sdk/client/query/tx.h index 7b7690d2b3..940f0d70c3 100644 --- a/include/ydb-cpp-sdk/client/query/tx.h +++ b/include/ydb-cpp-sdk/client/query/tx.h @@ -4,8 +4,6 @@ #include -#include - namespace NYdb::inline V3::NQuery { struct TTxOnlineSettings { @@ -85,35 +83,4 @@ struct TTxSettings { ETransactionMode Mode_; }; -struct TTxControl { - using TSelf = TTxControl; - - static TTxControl Tx(const std::string& txId) { - return TTxControl(txId); - } - - static TTxControl BeginTx(const TTxSettings& settings = TTxSettings()) { - return TTxControl(settings); - } - - static TTxControl NoTx() { - return TTxControl(); - } - - const std::optional TxId_; - const std::optional TxSettings_; - FLUENT_SETTING_FLAG(CommitTx); - - bool HasTx() const { return TxId_.has_value() || TxSettings_.has_value(); } - -private: - TTxControl() {} - - TTxControl(const std::string& txId) - : TxId_(txId) {} - - TTxControl(const TTxSettings& txSettings) - : TxSettings_(txSettings) {} -}; - } // namespace NYdb::NQuery diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index 5ca46f7c82..cce1895cdf 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -11,6 +11,7 @@ #include #include #include +#include #include @@ -1755,8 +1756,6 @@ struct TReadTableSettings : public TRequestSettings { FLUENT_SETTING_OPTIONAL(bool, ReturnNotNullAsOptional); }; -using TPrecommitTransactionCallback = std::function; - //! Represents all session operations //! Session is transparent logic representation of connection class TSession { @@ -1897,22 +1896,24 @@ TAsyncStatus TTableClient::RetryOperation( //////////////////////////////////////////////////////////////////////////////// //! Represents data transaction -class TTransaction { +class TTransaction : public TTransactionBase { friend class TTableClient; + public: - const std::string& GetId() const; bool IsActive() const; TAsyncCommitTransactionResult Commit(const TCommitTxSettings& settings = TCommitTxSettings()); TAsyncStatus Rollback(const TRollbackTxSettings& settings = TRollbackTxSettings()); TSession GetSession() const; - void AddPrecommitCallback(TPrecommitTransactionCallback cb); + void AddPrecommitCallback(TPrecommitTransactionCallback cb) override; + void AddOnFailureCallback(TOnFailureTransactionCallback cb) override; private: TTransaction(const TSession& session, const std::string& txId); TAsyncStatus Precommit() const; + NThreading::TFuture ProcessFailure() const; class TImpl; diff --git a/include/ydb-cpp-sdk/client/topic/read_session.h b/include/ydb-cpp-sdk/client/topic/read_session.h index 696d3f01f0..03f499743a 100644 --- a/include/ydb-cpp-sdk/client/topic/read_session.h +++ b/include/ydb-cpp-sdk/client/topic/read_session.h @@ -6,6 +6,7 @@ #include "retry_policy.h" #include +#include #include #include @@ -13,10 +14,6 @@ #include -namespace NYdb::inline V3::NTable { - class TTransaction; -} - namespace NYdb::inline V3::NTopic { //! Read settings for single topic. @@ -206,7 +203,7 @@ struct TReadSessionGetEventSettings : public TCommonClientSettingsBase::max()); - FLUENT_SETTING_OPTIONAL(std::reference_wrapper, Tx); + FLUENT_SETTING_OPTIONAL(std::reference_wrapper, Tx); }; class IReadSession { diff --git a/include/ydb-cpp-sdk/client/topic/write_session.h b/include/ydb-cpp-sdk/client/topic/write_session.h index 585e32f13c..68525b87fe 100644 --- a/include/ydb-cpp-sdk/client/topic/write_session.h +++ b/include/ydb-cpp-sdk/client/topic/write_session.h @@ -6,19 +6,14 @@ #include "retry_policy.h" #include "write_events.h" +#include #include #include #include -namespace NYdb::inline V3::NTable { - class TTransaction; -} - namespace NYdb::inline V3::NTopic { -using TTransaction = NTable::TTransaction; - //! Settings for write session. struct TWriteSessionSettings : public TRequestSettings { using TSelf = TWriteSessionSettings; @@ -192,9 +187,9 @@ struct TWriteMessage { FLUENT_SETTING(TMessageMeta, MessageMeta); //! Transaction id - FLUENT_SETTING_OPTIONAL(std::reference_wrapper, Tx); + FLUENT_SETTING_OPTIONAL(std::reference_wrapper, Tx); - TTransaction* GetTxPtr() const + TTransactionBase* GetTxPtr() const { return Tx_ ? &Tx_->get() : nullptr; } @@ -207,7 +202,7 @@ class ISimpleBlockingWriteSession : public TThrRefBase { //! return - true if write succeeded, false if message was not enqueued for write within blockTimeout. //! no Ack is provided. virtual bool Write(TWriteMessage&& message, - NTable::TTransaction* tx = nullptr, + TTransactionBase* tx = nullptr, const TDuration& blockTimeout = TDuration::Max()) = 0; @@ -254,7 +249,7 @@ class IWriteSession { //! Write single message. //! continuationToken - a token earlier provided to client with ReadyToAccept event. virtual void Write(TContinuationToken&& continuationToken, TWriteMessage&& message, - NTable::TTransaction* tx = nullptr) = 0; + TTransactionBase* tx = nullptr) = 0; //! Write single message. Old method with only basic message options. virtual void Write(TContinuationToken&& continuationToken, std::string_view data, std::optional seqNo = std::nullopt, @@ -263,7 +258,7 @@ class IWriteSession { //! Write single message that is already coded by codec. //! continuationToken - a token earlier provided to client with ReadyToAccept event. virtual void WriteEncoded(TContinuationToken&& continuationToken, TWriteMessage&& params, - NTable::TTransaction* tx = nullptr) = 0; + TTransactionBase* tx = nullptr) = 0; //! Write single message that is already compressed by codec. Old method with only basic message options. virtual void WriteEncoded(TContinuationToken&& continuationToken, std::string_view data, ECodec codec, uint32_t originalSize, diff --git a/include/ydb-cpp-sdk/client/types/tx/tx.h b/include/ydb-cpp-sdk/client/types/tx/tx.h new file mode 100644 index 0000000000..21c7afbfbe --- /dev/null +++ b/include/ydb-cpp-sdk/client/types/tx/tx.h @@ -0,0 +1,34 @@ +#pragma once + +#include + +#include + +namespace NYdb::inline V3 { + +using TPrecommitTransactionCallback = std::function; +using TOnFailureTransactionCallback = std::function()>; + +class TTransactionBase { +public: + const std::string& GetId() const { + return *TxId_; + } + + const std::string& GetSessionId() const { + return *SessionId_; + } + + virtual void AddPrecommitCallback(TPrecommitTransactionCallback cb) = 0; + virtual void AddOnFailureCallback(TOnFailureTransactionCallback cb) = 0; + + virtual ~TTransactionBase() = default; + +protected: + TTransactionBase() = default; + + const std::string* SessionId_ = nullptr; + const std::string* TxId_ = nullptr; +}; + +} // namespace NYdb diff --git a/src/client/federated_topic/impl/federated_write_session.h b/src/client/federated_topic/impl/federated_write_session.h index d9701662d9..31c0ed289d 100644 --- a/src/client/federated_topic/impl/federated_write_session.h +++ b/src/client/federated_topic/impl/federated_write_session.h @@ -173,13 +173,13 @@ class TFederatedWriteSession : public NTopic::IWriteSession, NThreading::TFuture GetInitSeqNo() override { return TryGetImpl()->GetInitSeqNo(); } - void Write(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& message, NTable::TTransaction* tx = nullptr) override { + void Write(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& message, TTransactionBase* tx = nullptr) override { if (tx) { ythrow yexception() << "transactions are not supported"; } TryGetImpl()->Write(std::move(continuationToken), std::move(message)); } - void WriteEncoded(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& params, NTable::TTransaction* tx = nullptr) override { + void WriteEncoded(NTopic::TContinuationToken&& continuationToken, NTopic::TWriteMessage&& params, TTransactionBase* tx = nullptr) override { if (tx) { ythrow yexception() << "transactions are not supported"; } diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 9888dd6115..4f5c7622ec 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -19,6 +19,8 @@ #include +#include + namespace NYdb::inline V3::NQuery { using TRetryContextResultAsync = NRetry::Async::TRetryContext; @@ -730,17 +732,165 @@ TAsyncBeginTransactionResult TSession::BeginTransaction(const TTxSettings& txSet Client_->Settings_.SessionPoolSettings_.CloseIdleThreshold_); } +class TTransaction::TImpl : public std::enable_shared_from_this { +public: + TImpl(const TSession& session, const std::string& txId) + : Session_(session) + , TxId_(txId) + {} + + const std::string& GetId() const { + return TxId_; + } + + const std::string& GetSessionId() const { + return Session_.GetId(); + } + + bool IsActive() const { + return !TxId_.empty(); + } + + TAsyncStatus Precommit() const { + TStatus status(EStatus::SUCCESS, {}); + + for (auto& callback : PrecommitCallbacks) { + if (!callback) { + continue; + } + + // If you send multiple requests in parallel, the `KQP` service can respond with `SESSION_BUSY`. + // Therefore, precommit operations are performed sequentially. Here we move the callback to a local variable, + // because otherwise it may be called twice. + auto localCallback = std::move(callback); + + if (!status.IsSuccess()) { + co_return status; + } + + status = co_await localCallback(); + } + + co_return status; + } + + NThreading::TFuture ProcessFailure() const { + for (auto& callback : OnFailureCallbacks) { + if (!callback) { + continue; + } + + // If you send multiple requests in parallel, the `KQP` service can respond with `SESSION_BUSY`. + // Therefore, precommit operations are performed sequentially. Here we move the callback to a local variable, + // because otherwise it may be called twice. + auto localCallback = std::move(callback); + + co_await localCallback(); + } + + co_return; + } + + TAsyncCommitTransactionResult Commit(const TCommitTxSettings& settings = TCommitTxSettings()) { + auto self = shared_from_this(); + + self->ChangesAreAccepted = false; + auto settingsCopy = settings; + + auto precommitResult = co_await self->Precommit(); + + if (!precommitResult.IsSuccess()) { + co_return TCommitTransactionResult(TStatus(precommitResult)); + } + + PrecommitCallbacks.clear(); + + auto commitResult = co_await self->Session_.Client_->CommitTransaction(self->TxId_, settingsCopy, self->Session_); + + if (!commitResult.IsSuccess()) { + co_await self->ProcessFailure(); + } + + co_return commitResult; + } + + TAsyncStatus Rollback(const TRollbackTxSettings& settings = TRollbackTxSettings()) { + auto self = shared_from_this(); + + self->ChangesAreAccepted = false; + + auto rollbackResult = co_await self->Session_.Client_->RollbackTransaction(self->TxId_, settings, self->Session_); + + co_await self->ProcessFailure(); + co_return rollbackResult; + } + + TSession GetSession() const { + return Session_; + } + + void AddPrecommitCallback(TPrecommitTransactionCallback cb) { + if (!ChangesAreAccepted) { + ythrow TContractViolation("Changes are no longer accepted"); + } + + PrecommitCallbacks.push_back(std::move(cb)); + } + + void AddOnFailureCallback(TOnFailureTransactionCallback cb) { + if (!ChangesAreAccepted) { + ythrow TContractViolation("Changes are no longer accepted"); + } + + OnFailureCallbacks.push_back(std::move(cb)); + } + + TSession Session_; + std::string TxId_; + +private: + bool ChangesAreAccepted = true; // haven't called Commit or Rollback yet + std::vector PrecommitCallbacks; + std::vector OnFailureCallbacks; +}; + TTransaction::TTransaction(const TSession& session, const std::string& txId) - : Session_(session) - , TxId_(txId) -{} + : TransactionImpl_(std::make_shared(session, txId)) +{ + SessionId_ = &TransactionImpl_->Session_.GetId(); + TxId_ = &TransactionImpl_->TxId_; +} + +bool TTransaction::IsActive() const { + return TransactionImpl_->IsActive(); +} + +TAsyncStatus TTransaction::Precommit() const { + return TransactionImpl_->Precommit(); +} + +NThreading::TFuture TTransaction::ProcessFailure() const { + return TransactionImpl_->ProcessFailure(); +} -TAsyncCommitTransactionResult TTransaction::Commit(const NYdb::NQuery::TCommitTxSettings& settings) { - return Session_.Client_->CommitTransaction(TxId_, settings, Session_); +TAsyncCommitTransactionResult TTransaction::Commit(const TCommitTxSettings& settings) { + return TransactionImpl_->Commit(settings); } TAsyncStatus TTransaction::Rollback(const TRollbackTxSettings& settings) { - return Session_.Client_->RollbackTransaction(TxId_, settings, Session_); + return TransactionImpl_->Rollback(settings); +} + +TSession TTransaction::GetSession() const { + return TransactionImpl_->GetSession(); +} + +void TTransaction::AddPrecommitCallback(TPrecommitTransactionCallback cb) { + TransactionImpl_->AddPrecommitCallback(std::move(cb)); +} + +void TTransaction::AddOnFailureCallback(TOnFailureTransactionCallback cb) { + TransactionImpl_->AddOnFailureCallback(std::move(cb)); } TBeginTransactionResult::TBeginTransactionResult(TStatus&& status, TTransaction transaction) diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index a91bc7c4b0..852f5cf87e 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -13,6 +13,8 @@ #include +#include + namespace NYdb::inline V3::NQuery { using namespace NThreading; @@ -221,100 +223,127 @@ struct TExecuteQueryBuffer : public TThrRefBase, TNonCopyable { } }; -TFuture> StreamExecuteQueryImpl( - const std::shared_ptr& connections, const TDbDriverStatePtr& driverState, - const std::string& query, const TTxControl& txControl, const ::google::protobuf::Map* params, - const TExecuteQuerySettings& settings, const std::optional& session) -{ - auto request = MakeRequest(); - request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); - request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); - request.set_pool_id(TStringType{settings.ResourcePool_}); - request.mutable_query_content()->set_text(TStringType{query}); - request.mutable_query_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); - if (session.has_value()) { - request.set_session_id(TStringType{session->GetId()}); - } else if ((txControl.TxSettings_.has_value() && !txControl.CommitTx_) || txControl.TxId_.has_value()) { - throw TContractViolation("Interactive tx must use explisit session"); - } +class TExecQueryInternal { +public: + static TFuture> ExecuteQueryCommon( + const std::shared_ptr& connections, const TDbDriverStatePtr& driverState, + const std::string& query, const TTxControl& txControl, const ::google::protobuf::Map* params, + const TExecuteQuerySettings& settings, const std::optional& session) + { + auto request = MakeRequest(); + request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); + request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); + request.set_pool_id(TStringType{settings.ResourcePool_}); + request.mutable_query_content()->set_text(TStringType{query}); + request.mutable_query_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); + if (session.has_value()) { + request.set_session_id(TStringType{session->GetId()}); + } else if ((std::holds_alternative(txControl.Tx_) && !txControl.CommitTx_) || + std::holds_alternative(txControl.Tx_) || + std::holds_alternative(txControl.Tx_)) { + throw TContractViolation("Interactive tx must use explisit session"); + } - if (settings.ConcurrentResultSets_) { - request.set_concurrent_result_sets(*settings.ConcurrentResultSets_); - } + if (settings.ConcurrentResultSets_) { + request.set_concurrent_result_sets(*settings.ConcurrentResultSets_); + } - if (settings.OutputChunkMaxSize_) { - request.set_response_part_limit_bytes(*settings.OutputChunkMaxSize_); - } + if (settings.OutputChunkMaxSize_) { + request.set_response_part_limit_bytes(*settings.OutputChunkMaxSize_); + } - if (settings.StatsCollectPeriod_) { - request.set_stats_period_ms(settings.StatsCollectPeriod_->count()); - } + if (settings.StatsCollectPeriod_) { + request.set_stats_period_ms(settings.StatsCollectPeriod_->count()); + } + + if (txControl.HasTx()) { + auto requestTxControl = request.mutable_tx_control(); + requestTxControl->set_commit_tx(txControl.CommitTx_); - if (txControl.HasTx()) { - auto requestTxControl = request.mutable_tx_control(); - requestTxControl->set_commit_tx(txControl.CommitTx_); - if (txControl.TxId_) { - requestTxControl->set_tx_id(TStringType{txControl.TxId_.value()}); + if (auto* tx = std::get_if(&txControl.Tx_)) { + requestTxControl->set_tx_id(TStringType{tx->GetId()}); + } else if (auto* txId = std::get_if(&txControl.Tx_)) { + requestTxControl->set_tx_id(TStringType{*txId}); + } else if (auto* txSettings = std::get_if(&txControl.Tx_)) { + SetTxSettings(*txSettings, requestTxControl->mutable_begin_tx()); + } else { + Y_DEBUG_ABORT("Unexpected tx control type"); + } } else { - Y_ASSERT(txControl.TxSettings_); - SetTxSettings(*txControl.TxSettings_, requestTxControl->mutable_begin_tx()); + Y_ASSERT(!txControl.CommitTx_); } - } else { - Y_ASSERT(!txControl.CommitTx_); - } - if (params) { - *request.mutable_parameters() = *params; - } + if (params) { + *request.mutable_parameters() = *params; + } - auto promise = NewPromise>(); + auto promise = NewPromise>(); - auto rpcSettings = TRpcRequestSettings::Make(settings); - if (session.has_value()) { - rpcSettings.PreferredEndpoint = TEndpointKey(GetNodeIdFromSession(session->GetId())); - } + auto rpcSettings = TRpcRequestSettings::Make(settings); + if (session.has_value()) { + rpcSettings.PreferredEndpoint = TEndpointKey(GetNodeIdFromSession(session->GetId())); + } - connections->StartReadStream< - Ydb::Query::V1::QueryService, - Ydb::Query::ExecuteQueryRequest, - Ydb::Query::ExecuteQueryResponsePart> - ( - std::move(request), - [promise] (TPlainStatus status, TExecuteQueryProcessorPtr processor) mutable { - promise.SetValue(std::make_pair(status, processor)); - }, - &Ydb::Query::V1::QueryService::Stub::AsyncExecuteQuery, - driverState, - rpcSettings - ); + connections->StartReadStream< + Ydb::Query::V1::QueryService, + Ydb::Query::ExecuteQueryRequest, + Ydb::Query::ExecuteQueryResponsePart> + ( + std::move(request), + [promise] (TPlainStatus status, TExecuteQueryProcessorPtr processor) mutable { + promise.SetValue(std::make_pair(status, processor)); + }, + &Ydb::Query::V1::QueryService::Stub::AsyncExecuteQuery, + driverState, + rpcSettings + ); - return promise.GetFuture(); -} + return promise.GetFuture(); + } + +}; TAsyncExecuteQueryIterator TExecQueryImpl::StreamExecuteQuery(const std::shared_ptr& connections, const TDbDriverStatePtr& driverState, const std::string& query, const TTxControl& txControl, const std::optional& params, const TExecuteQuerySettings& settings, const std::optional& session) { - auto promise = NewPromise(); - - auto iteratorCallback = [promise, session](TFuture> future) mutable { - Y_ASSERT(future.HasValue()); - auto pair = future.ExtractValue(); - promise.SetValue(TExecuteQueryIterator( - pair.second - ? std::make_shared(pair.second, pair.first.Endpoint, session) - : nullptr, - std::move(pair.first)) - ); - }; + TPlainStatus plainStatus; + TExecuteQueryProcessorPtr processor; + + auto sessionCopy = session; + + if (auto* txPtr = std::get_if(&txControl.Tx_); txPtr && txControl.CommitTx_) { + auto queryCopy = query; + auto txControlCopy = txControl; + auto paramsCopy = params; + auto settingsCopy = settings; - auto paramsProto = params - ? ¶ms->GetProtoMap() - : nullptr; + auto tx = *txPtr; + auto precommitStatus = co_await tx.Precommit(); - StreamExecuteQueryImpl(connections, driverState, query, txControl, paramsProto, settings, session) - .Subscribe(iteratorCallback); - return promise.GetFuture(); + if (!precommitStatus.IsSuccess()) { + co_return TExecuteQueryIterator(nullptr, std::move(precommitStatus)); + } + + std::tie(plainStatus, processor) = co_await TExecQueryInternal::ExecuteQueryCommon( + connections, driverState, queryCopy, txControlCopy, paramsCopy ? ¶msCopy->GetProtoMap() : nullptr, settingsCopy, sessionCopy); + + if (!plainStatus.Ok()) { + co_await tx.ProcessFailure(); + + co_return TExecuteQueryIterator(nullptr, std::move(plainStatus)); + } + } else { + std::tie(plainStatus, processor) = co_await AsExtractingAwaitable(TExecQueryInternal::ExecuteQueryCommon( + connections, driverState, query, txControl, params ? ¶ms->GetProtoMap() : nullptr, settings, sessionCopy)); + } + + co_return TExecuteQueryIterator( + processor + ? std::make_shared(processor, plainStatus.Endpoint, sessionCopy) + : nullptr, + std::move(plainStatus) + ); } TAsyncExecuteQueryResult TExecQueryImpl::ExecuteQuery(const std::shared_ptr& connections, diff --git a/src/client/table/impl/table_client.h b/src/client/table/impl/table_client.h index 4cd1118347..1c93045f52 100644 --- a/src/client/table/impl/table_client.h +++ b/src/client/table/impl/table_client.h @@ -18,6 +18,8 @@ #include "request_migrator.h" #include "readers.h" +#include + namespace NYdb::inline V3 { namespace NTable { @@ -179,24 +181,30 @@ class TTableClient::TImpl: public TClientImplCommon, public const TExecDataQuerySettings& settings, bool fromCache ) { if (!txControl.Tx_.has_value() || !txControl.CommitTx_) { - return ExecuteDataQueryInternal(session, query, txControl, params, settings, fromCache); + co_return co_await AsExtractingAwaitable(ExecuteDataQueryInternal(session, query, txControl, params, settings, fromCache)); } - auto onPrecommitCompleted = [this, session, query, txControl, params, settings, fromCache](const NThreading::TFuture& f) { - TStatus status = f.GetValueSync(); - if (!status.IsSuccess()) { - return NThreading::MakeFuture(TDataQueryResult(std::move(status), - {}, - txControl.Tx_, - std::nullopt, - false, - std::nullopt)); - } + auto sessionCopy = session; + auto settingsCopy = settings; + auto queryCopy = query; + auto txControlCopy = txControl; + auto paramsCopy = params; - return ExecuteDataQueryInternal(session, query, txControl, params, settings, fromCache); - }; + auto status = co_await txControlCopy.Tx_->Precommit(); + + if (!status.IsSuccess()) { + co_return TDataQueryResult(std::move(status), {}, txControlCopy.Tx_, std::nullopt, false, std::nullopt); + } + + auto dataQueryResult = co_await AsExtractingAwaitable(ExecuteDataQueryInternal(sessionCopy, queryCopy, txControlCopy, paramsCopy, settingsCopy, fromCache)); + + if (!dataQueryResult.IsSuccess()) { + co_await txControlCopy.Tx_->ProcessFailure(); + + co_return std::move(dataQueryResult); + } - return txControl.Tx_->Precommit().Apply(onPrecommitCompleted); + co_return std::move(dataQueryResult); } template diff --git a/src/client/table/impl/transaction.cpp b/src/client/table/impl/transaction.cpp index 6627804a18..0c0340abb1 100644 --- a/src/client/table/impl/transaction.cpp +++ b/src/client/table/impl/transaction.cpp @@ -11,7 +11,7 @@ TTransaction::TImpl::TImpl(const TSession& session, const std::string& txId) TAsyncStatus TTransaction::TImpl::Precommit() const { - auto result = NThreading::MakeFuture(TStatus(EStatus::SUCCESS, {})); + TStatus status(EStatus::SUCCESS, {}); for (auto& callback : PrecommitCallbacks) { if (!callback) { @@ -19,47 +19,71 @@ TAsyncStatus TTransaction::TImpl::Precommit() const } // If you send multiple requests in parallel, the `KQP` service can respond with `SESSION_BUSY`. - // Therefore, precommit operations are performed sequentially. Here we capture the closure to - // trigger it later. - auto action = [callback = std::move(callback)](const TAsyncStatus& prev) { - if (const TStatus& status = prev.GetValue(); !status.IsSuccess()) { - return prev; - } + // Therefore, precommit operations are performed sequentially. Here we move the callback to a local variable, + // because otherwise it may be called twice. + auto localCallback = std::move(callback); - return callback(); - }; + if (!status.IsSuccess()) { + co_return status; + } - result = result.Apply(action); + status = co_await localCallback(); } - return result; + co_return status; +} + +NThreading::TFuture TTransaction::TImpl::ProcessFailure() const +{ + for (auto& callback : OnFailureCallbacks) { + if (!callback) { + continue; + } + + // If you send multiple requests in parallel, the `KQP` service can respond with `SESSION_BUSY`. + // Therefore, precommit operations are performed sequentially. Here we move the callback to a local variable, + // because otherwise it may be called twice. + auto localCallback = std::move(callback); + + co_await localCallback(); + } + + co_return; } TAsyncCommitTransactionResult TTransaction::TImpl::Commit(const TCommitTxSettings& settings) { - ChangesAreAccepted = false; + auto self = shared_from_this(); - auto result = Precommit(); + self->ChangesAreAccepted = false; + auto settingsCopy = settings; - auto precommitsCompleted = [this, settings](const TAsyncStatus& result) mutable { - if (const TStatus& status = result.GetValue(); !status.IsSuccess()) { - return NThreading::MakeFuture(TCommitTransactionResult(TStatus(status), std::nullopt)); - } + auto precommitResult = co_await self->Precommit(); - PrecommitCallbacks.clear(); + if (!precommitResult.IsSuccess()) { + co_return TCommitTransactionResult(TStatus(precommitResult), std::nullopt); + } + + self->PrecommitCallbacks.clear(); - return Session_.Client_->CommitTransaction(Session_, - TxId_, - settings); - }; + auto commitResult = co_await self->Session_.Client_->CommitTransaction(self->Session_, self->TxId_, settingsCopy); + + if (!commitResult.IsSuccess()) { + co_await self->ProcessFailure(); + } - return result.Apply(precommitsCompleted); + co_return commitResult; } TAsyncStatus TTransaction::TImpl::Rollback(const TRollbackTxSettings& settings) { - ChangesAreAccepted = false; - return Session_.Client_->RollbackTransaction(Session_, TxId_, settings); + auto self = shared_from_this(); + self->ChangesAreAccepted = false; + + auto rollbackResult = co_await self->Session_.Client_->RollbackTransaction(self->Session_, self->TxId_, settings); + + co_await self->ProcessFailure(); + co_return rollbackResult; } void TTransaction::TImpl::AddPrecommitCallback(TPrecommitTransactionCallback cb) @@ -71,4 +95,13 @@ void TTransaction::TImpl::AddPrecommitCallback(TPrecommitTransactionCallback cb) PrecommitCallbacks.push_back(std::move(cb)); } +void TTransaction::TImpl::AddOnFailureCallback(TOnFailureTransactionCallback cb) +{ + if (!ChangesAreAccepted) { + ythrow TContractViolation("Changes are no longer accepted"); + } + + OnFailureCallbacks.push_back(std::move(cb)); +} + } diff --git a/src/client/table/impl/transaction.h b/src/client/table/impl/transaction.h index c226d8ca35..f5b6025aae 100644 --- a/src/client/table/impl/transaction.h +++ b/src/client/table/impl/transaction.h @@ -4,7 +4,7 @@ namespace NYdb::inline V3::NTable { -class TTransaction::TImpl { +class TTransaction::TImpl : public std::enable_shared_from_this { public: TImpl(const TSession& session, const std::string& txId); @@ -12,11 +12,17 @@ class TTransaction::TImpl { return TxId_; } + const std::string& GetSessionId() const { + return Session_.GetId(); + } + bool IsActive() const { return !TxId_.empty(); } TAsyncStatus Precommit() const; + NThreading::TFuture ProcessFailure() const; + TAsyncCommitTransactionResult Commit(const TCommitTxSettings& settings = TCommitTxSettings()); TAsyncStatus Rollback(const TRollbackTxSettings& settings = TRollbackTxSettings()); @@ -25,13 +31,15 @@ class TTransaction::TImpl { } void AddPrecommitCallback(TPrecommitTransactionCallback cb); + void AddOnFailureCallback(TOnFailureTransactionCallback cb); -private: TSession Session_; std::string TxId_; +private: bool ChangesAreAccepted = true; // haven't called Commit or Rollback yet mutable std::vector PrecommitCallbacks; + mutable std::vector OnFailureCallbacks; }; } diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index ae962207a4..37810f48f7 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -1983,12 +1983,10 @@ TTxControl::TTxControl(const TTxSettings& begin) //////////////////////////////////////////////////////////////////////////////// TTransaction::TTransaction(const TSession& session, const std::string& txId) - : TransactionImpl_(new TTransaction::TImpl(session, txId)) -{} - -const std::string& TTransaction::GetId() const + : TransactionImpl_(std::make_shared(session, txId)) { - return TransactionImpl_->GetId(); + SessionId_ = &TransactionImpl_->Session_.GetId(); + TxId_ = &TransactionImpl_->TxId_; } bool TTransaction::IsActive() const @@ -2001,11 +1999,18 @@ TAsyncStatus TTransaction::Precommit() const return TransactionImpl_->Precommit(); } -TAsyncCommitTransactionResult TTransaction::Commit(const TCommitTxSettings& settings) { +NThreading::TFuture TTransaction::ProcessFailure() const +{ + return TransactionImpl_->ProcessFailure(); +} + +TAsyncCommitTransactionResult TTransaction::Commit(const TCommitTxSettings& settings) +{ return TransactionImpl_->Commit(settings); } -TAsyncStatus TTransaction::Rollback(const TRollbackTxSettings& settings) { +TAsyncStatus TTransaction::Rollback(const TRollbackTxSettings& settings) +{ return TransactionImpl_->Rollback(settings); } @@ -2019,6 +2024,10 @@ void TTransaction::AddPrecommitCallback(TPrecommitTransactionCallback cb) TransactionImpl_->AddPrecommitCallback(std::move(cb)); } +void TTransaction::AddOnFailureCallback(TOnFailureTransactionCallback cb) { + TransactionImpl_->AddOnFailureCallback(std::move(cb)); +} + //////////////////////////////////////////////////////////////////////////////// TDataQuery::TDataQuery(const TSession& session, const std::string& text, const std::string& id) diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index 5ef33c0769..72add15415 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -1164,10 +1164,10 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContextSetCallbackContext(TEnableSelfContext>::SelfContext); } - void CollectOffsets(NTable::TTransaction& tx, + void CollectOffsets(TTransactionBase& tx, const std::vector& events, std::shared_ptr client); - void CollectOffsets(NTable::TTransaction& tx, + void CollectOffsets(TTransactionBase& tx, const TReadSessionEvent::TEvent& event, std::shared_ptr client); @@ -1326,7 +1326,7 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext; using TTransactionMap = std::unordered_map>; - void TrySubscribeOnTransactionCommit(NTable::TTransaction& tx, + void TrySubscribeOnTransactionCommit(TTransactionBase& tx, std::shared_ptr client); TTransactionInfoPtr GetOrCreateTxInfo(const TTransactionId& txId); void DeleteTx(const TTransactionId& txId); diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index b9e19b4d36..28bd4a50e6 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -1919,7 +1919,7 @@ void TSingleClusterReadSessionImpl::ConfirmPartitionStream } template -void TSingleClusterReadSessionImpl::CollectOffsets(NTable::TTransaction& tx, +void TSingleClusterReadSessionImpl::CollectOffsets(TTransactionBase& tx, const std::vector& events, std::shared_ptr client) { @@ -1931,7 +1931,7 @@ void TSingleClusterReadSessionImpl::CollectOffsets(NTable: } template -void TSingleClusterReadSessionImpl::CollectOffsets(NTable::TTransaction& tx, +void TSingleClusterReadSessionImpl::CollectOffsets(TTransactionBase& tx, const TReadSessionEvent::TEvent& event, std::shared_ptr client) { @@ -1943,7 +1943,7 @@ void TSingleClusterReadSessionImpl::CollectOffsets(NTable: } template -void TSingleClusterReadSessionImpl::TrySubscribeOnTransactionCommit(NTable::TTransaction& tx, +void TSingleClusterReadSessionImpl::TrySubscribeOnTransactionCommit(TTransactionBase& tx, std::shared_ptr client) { const TTransactionId txId = MakeTransactionId(tx); diff --git a/src/client/topic/impl/transaction.cpp b/src/client/topic/impl/transaction.cpp index 662a4e6121..3932c5c51d 100644 --- a/src/client/topic/impl/transaction.cpp +++ b/src/client/topic/impl/transaction.cpp @@ -1,11 +1,10 @@ #include "transaction.h" -#include namespace NYdb::inline V3::NTopic { -TTransactionId MakeTransactionId(const NTable::TTransaction& tx) +TTransactionId MakeTransactionId(const TTransactionBase& tx) { - return {tx.GetSession().GetId(), tx.GetId()}; + return {tx.GetSessionId(), tx.GetId()}; } TStatus MakeStatus(EStatus code, NYdb::NIssue::TIssues&& issues) diff --git a/src/client/topic/impl/transaction.h b/src/client/topic/impl/transaction.h index c685e50f92..806684e749 100644 --- a/src/client/topic/impl/transaction.h +++ b/src/client/topic/impl/transaction.h @@ -1,12 +1,6 @@ #pragma once -#include - -namespace NYdb::inline V3::NTable { - -class TTransaction; - -} +#include namespace NYdb::inline V3::NTopic { @@ -27,7 +21,7 @@ bool operator!=(const TTransactionId& lhs, const TTransactionId& rhs) return !(lhs == rhs); } -TTransactionId MakeTransactionId(const NTable::TTransaction& tx); +TTransactionId MakeTransactionId(const TTransactionBase& tx); TStatus MakeSessionExpiredError(); TStatus MakeCommitTransactionSuccess(); diff --git a/src/client/topic/impl/write_session.cpp b/src/client/topic/impl/write_session.cpp index b19d03eae4..5f2f54605b 100644 --- a/src/client/topic/impl/write_session.cpp +++ b/src/client/topic/impl/write_session.cpp @@ -46,7 +46,7 @@ void TWriteSession::WriteEncoded(TContinuationToken&& token, std::string_view da } void TWriteSession::WriteEncoded(TContinuationToken&& token, TWriteMessage&& message, - NTable::TTransaction* tx) + TTransactionBase* tx) { if (tx) { message.Tx(*tx); @@ -65,7 +65,7 @@ void TWriteSession::Write(TContinuationToken&& token, std::string_view data, std } void TWriteSession::Write(TContinuationToken&& token, TWriteMessage&& message, - NTable::TTransaction* tx) { + TTransactionBase* tx) { if (tx) { message.Tx(*tx); } @@ -124,7 +124,7 @@ bool TSimpleBlockingWriteSession::Write( } bool TSimpleBlockingWriteSession::Write( - TWriteMessage&& message, NTable::TTransaction* tx, const TDuration& blockTimeout + TWriteMessage&& message, TTransactionBase* tx, const TDuration& blockTimeout ) { auto continuationToken = WaitForToken(blockTimeout); if (continuationToken.has_value()) { diff --git a/src/client/topic/impl/write_session.h b/src/client/topic/impl/write_session.h index 77beb2898d..3112fd1361 100644 --- a/src/client/topic/impl/write_session.h +++ b/src/client/topic/impl/write_session.h @@ -37,10 +37,10 @@ class TWriteSession : public IWriteSession, std::optional seqNo = std::nullopt, std::optional createTimestamp = std::nullopt) override; void Write(TContinuationToken&& continuationToken, TWriteMessage&& message, - NTable::TTransaction* tx = nullptr) override; + TTransactionBase* tx = nullptr) override; void WriteEncoded(TContinuationToken&& continuationToken, TWriteMessage&& message, - NTable::TTransaction* tx = nullptr) override; + TTransactionBase* tx = nullptr) override; NThreading::TFuture WaitEvent() override; @@ -70,7 +70,7 @@ class TSimpleBlockingWriteSession : public ISimpleBlockingWriteSession { const TDuration& blockTimeout = TDuration::Max()) override; bool Write(TWriteMessage&& message, - NTable::TTransaction* tx = nullptr, + TTransactionBase* tx = nullptr, const TDuration& blockTimeout = TDuration::Max()) override; uint64_t GetInitSeqNo() override; diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index 21ea692a4d..55bc19178e 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -50,13 +50,13 @@ TTxIdOpt GetTransactionId(const std::optional& tx) return TTxId(tx->SessionId, tx->TxId); } -std::optional MakeTransactionId(const NTable::TTransaction* tx) +std::optional MakeTransactionId(const TTransactionBase* tx) { if (!tx) { return std::nullopt; } - return TTransactionId{tx->GetSession().GetId(), tx->GetId()}; + return TTransactionId{tx->GetSessionId(), tx->GetId()}; } } @@ -537,7 +537,7 @@ NThreading::TFuture TWriteSessionImpl::WaitEvent() { return EventsQueue->WaitEvent(); } -void TWriteSessionImpl::TrySubscribeOnTransactionCommit(TTransaction* tx) +void TWriteSessionImpl::TrySubscribeOnTransactionCommit(TTransactionBase* tx) { if (!tx) { return; diff --git a/src/client/topic/impl/write_session_impl.h b/src/client/topic/impl/write_session_impl.h index 6fc44d20ec..0434ed07e6 100644 --- a/src/client/topic/impl/write_session_impl.h +++ b/src/client/topic/impl/write_session_impl.h @@ -425,7 +425,7 @@ class TWriteSessionImpl : public TContinuationTokenIssuer, bool TxIsChanged(const Ydb::Topic::StreamWriteMessage_WriteRequest* writeRequest) const; - void TrySubscribeOnTransactionCommit(TTransaction* tx); + void TrySubscribeOnTransactionCommit(TTransactionBase* tx); void CancelTransactions(); TTransactionInfoPtr GetOrCreateTxInfo(const TTransactionId& txId); void TrySignalAllAcksReceived(ui64 seqNo); diff --git a/src/client/topic/ut/topic_to_table_ut.cpp b/src/client/topic/ut/topic_to_table_ut.cpp index c7705af45f..f2ce3569aa 100644 --- a/src/client/topic/ut/topic_to_table_ut.cpp +++ b/src/client/topic/ut/topic_to_table_ut.cpp @@ -2,8 +2,11 @@ #include #include +#include #include +#include + #include #include #include @@ -17,6 +20,9 @@ #include #include #include +#include +#include +#include namespace NYdb::NTopic::NTests { @@ -36,13 +42,13 @@ class TFixture : public NUnitTest::TBaseFixture { struct TTopicWriteSessionContext { TTopicWriteSessionPtr Session; - TMaybe ContinuationToken; + std::optional ContinuationToken; size_t WriteCount = 0; size_t WrittenAckCount = 0; size_t WrittenInTxAckCount = 0; void WaitForContinuationToken(); - void Write(const TString& message, NTable::TTransaction* tx = nullptr); + void Write(const std::string& message, TTransactionBase* tx = nullptr); size_t AckCount() const { return WrittenAckCount + WrittenInTxAckCount; } @@ -53,33 +59,54 @@ class TFixture : public NUnitTest::TBaseFixture { bool EnablePQConfigTransactionsAtSchemeShard = true; }; + class ISession { + public: + using TExecuteInTxResult = std::pair, std::unique_ptr>; + + virtual std::vector Execute(const std::string& query, + TTransactionBase* tx, + bool commit = true, + const TParams& params = TParamsBuilder().Build()) = 0; + + virtual TExecuteInTxResult ExecuteInTx(const std::string& query, + bool commit = true, + const TParams& params = TParamsBuilder().Build()) = 0; + + virtual std::unique_ptr BeginTx() = 0; + virtual void CommitTx(TTransactionBase& tx, EStatus status = EStatus::SUCCESS) = 0; + virtual void RollbackTx(TTransactionBase& tx, EStatus status = EStatus::SUCCESS) = 0; + + virtual void Close() = 0; + + virtual TAsyncStatus AsyncCommitTx(TTransactionBase& tx) = 0; + + virtual ~ISession() = default; + }; + void SetUp(NUnitTest::TTestContext&) override; void NotifySchemeShard(const TFeatureFlags& flags); - NTable::TSession CreateTableSession(); - NTable::TTransaction BeginTx(NTable::TSession& session); - void CommitTx(NTable::TTransaction& tx, EStatus status = EStatus::SUCCESS); - void RollbackTx(NTable::TTransaction& tx, EStatus status = EStatus::SUCCESS); + std::unique_ptr CreateSession(); TTopicReadSessionPtr CreateReader(); - void StartPartitionSession(TTopicReadSessionPtr reader, NTable::TTransaction& tx, ui64 offset); + void StartPartitionSession(TTopicReadSessionPtr reader, TTransactionBase& tx, ui64 offset); void StartPartitionSession(TTopicReadSessionPtr reader, ui64 offset); struct TReadMessageSettings { - NTable::TTransaction& Tx; + TTransactionBase& Tx; bool CommitOffsets = false; std::optional Offset; }; - void ReadMessage(TTopicReadSessionPtr reader, NTable::TTransaction& tx, ui64 offset); + void ReadMessage(TTopicReadSessionPtr reader, TTransactionBase& tx, ui64 offset); void ReadMessage(TTopicReadSessionPtr reader, const TReadMessageSettings& settings); void WriteMessage(const TString& message); void WriteMessages(const TVector& messages, const TString& topic, const TString& groupId, - NTable::TTransaction& tx); + TTransactionBase& tx); void CreateTopic(const TString& path = TString{TEST_TOPIC}, const TString& consumer = TEST_CONSUMER, @@ -117,12 +144,12 @@ class TFixture : public NUnitTest::TBaseFixture { void WriteToTopic(const TString& topicPath, const TString& messageGroupId, const TString& message, - NTable::TTransaction* tx = nullptr, + TTransactionBase* tx = nullptr, std::optional partitionId = std::nullopt); TVector ReadFromTopic(const TString& topicPath, const TString& consumerName, const TDuration& duration, - NTable::TTransaction* tx = nullptr, + TTransactionBase* tx = nullptr, TMaybe partitionId = Nothing()); void WaitForAcks(const TString& topicPath, const TString& messageGroupId, @@ -172,7 +199,8 @@ class TFixture : public NUnitTest::TBaseFixture { void CreateTable(const TString& path); void WriteToTable(const TString& tablePath, const TVector& records, - NTable::TTransaction* tx); + ISession& session, + TTransactionBase* tx); size_t GetTableRecordsCount(const TString& tablePath); enum ERestartPQTabletMode { @@ -193,34 +221,115 @@ class TFixture : public NUnitTest::TBaseFixture { void WriteMessagesInTx(size_t big, size_t small); const TDriver& GetDriver() const; + NTable::TTableClient& GetTableClient(); void CheckTabletKeys(const TString& topicName); void DumpPQTabletKeys(const TString& topicName); - NTable::TDataQueryResult ExecuteDataQuery(NTable::TSession session, const TString& query, const NTable::TTxControl& control); - TVector Read_Exactly_N_Messages_From_Topic(const TString& topicPath, const TString& consumerName, size_t count); + void TestSessionAbort(); + + void TestTwoSessionOneConsumer(); + + void TestOffsetsCannotBePromotedWhenReadingInATransaction(); + + void TestWriteToTopicTwoWriteSession(); + + void TestWriteRandomSizedMessagesInWideTransactions(); + + void TestWriteOnlyBigMessagesInWideTransactions(); + + void TestTransactionsConflictOnSeqNo(); + void TestWriteToTopic1(); + void TestWriteToTopic2(); + + void TestWriteToTopic3(); + void TestWriteToTopic4(); + void TestWriteToTopic5(); + + void TestWriteToTopic6(); + void TestWriteToTopic7(); + void TestWriteToTopic8(); + void TestWriteToTopic9(); void TestWriteToTopic10(); void TestWriteToTopic11(); + void TestWriteToTopic12(); + + void TestWriteToTopic13(); + + void TestWriteToTopic14(); + + void TestWriteToTopic15(); + + void TestWriteToTopic16(); + + void TestWriteToTopic17(); + void TestWriteToTopic24(); + void TestWriteToTopic25(); + void TestWriteToTopic26(); void TestWriteToTopic27(); + void TestWriteToTopic28(); + + void TestWriteToTopic29(); + + void TestWriteToTopic30(); + + void TestWriteToTopic31(); + + void TestWriteToTopic32(); + + void TestWriteToTopic33(); + + void TestWriteToTopic34(); + + void TestWriteToTopic35(); + + void TestWriteToTopic36(); + + void TestWriteToTopic37(); + + void TestWriteToTopic38(); + + void TestWriteToTopic39(); + + void TestWriteToTopic40(); + + void TestWriteToTopic41(); + + void TestWriteToTopic42(); + + void TestWriteToTopic43(); + + void TestWriteToTopic44(); + + void TestWriteToTopic45(); + + void TestWriteToTopic46(); + + void TestWriteToTopic47(); + + void TestWriteToTopic48(); + + void TestWriteToTopic50(); + struct TAvgWriteBytes { ui64 PerSec = 0; ui64 PerMin = 0; @@ -246,9 +355,76 @@ class TFixture : public NUnitTest::TBaseFixture { size_t GetPQCacheRenameKeysCount(); + enum class EClientType { + Table, + Query, + None + }; + + virtual EClientType GetClientType() const = 0; + virtual ~TFixture() = default; + private: + class TTableSession : public ISession { + public: + TTableSession(NTable::TTableClient& client); + + std::vector Execute(const std::string& query, + TTransactionBase* tx, + bool commit = true, + const TParams& params = TParamsBuilder().Build()) override; + + TExecuteInTxResult ExecuteInTx(const std::string& query, + bool commit = true, + const TParams& params = TParamsBuilder().Build()) override; + + std::unique_ptr BeginTx() override; + void CommitTx(TTransactionBase& tx, EStatus status = EStatus::SUCCESS) override; + void RollbackTx(TTransactionBase& tx, EStatus status = EStatus::SUCCESS) override; + + void Close() override; + + TAsyncStatus AsyncCommitTx(TTransactionBase& tx) override; + + private: + NTable::TSession Init(NTable::TTableClient& client); + + NTable::TSession Session_; + }; + + class TQuerySession : public ISession { + public: + TQuerySession(NQuery::TQueryClient& client, + const std::string& endpoint, + const std::string& database); + + std::vector Execute(const std::string& query, + TTransactionBase* tx, + bool commit = true, + const TParams& params = TParamsBuilder().Build()) override; + + TExecuteInTxResult ExecuteInTx(const std::string& query, + bool commit = true, + const TParams& params = TParamsBuilder().Build()) override; + + std::unique_ptr BeginTx() override; + void CommitTx(TTransactionBase& tx, EStatus status = EStatus::SUCCESS) override; + void RollbackTx(TTransactionBase& tx, EStatus status = EStatus::SUCCESS) override; + + void Close() override; + + TAsyncStatus AsyncCommitTx(TTransactionBase& tx) override; + + private: + NQuery::TSession Init(NQuery::TQueryClient& client); + + NQuery::TSession Session_; + std::string Endpoint_; + std::string Database_; + }; + template - E ReadEvent(TTopicReadSessionPtr reader, NTable::TTransaction& tx); + E ReadEvent(TTopicReadSessionPtr reader, TTransactionBase& tx); template E ReadEvent(TTopicReadSessionPtr reader); @@ -273,6 +449,8 @@ class TFixture : public NUnitTest::TBaseFixture { std::unique_ptr Setup; std::unique_ptr Driver; + std::unique_ptr TableClient; + std::unique_ptr QueryClient; THashMap, TTopicWriteSessionContext> TopicWriteSessions; THashMap TopicReadSessions; @@ -280,6 +458,27 @@ class TFixture : public NUnitTest::TBaseFixture { ui64 SchemaTxId = 1000; }; +class TFixtureTable : public TFixture { +protected: + EClientType GetClientType() const override { + return EClientType::Table; + } +}; + +class TFixtureQuery : public TFixture { +protected: + EClientType GetClientType() const override { + return EClientType::Query; + } +}; + +class TFixtureNoClient : public TFixture { +protected: + EClientType GetClientType() const override { + return EClientType::None; + } +}; + TFixture::TTableRecord::TTableRecord(const TString& key, const TString& value) : Key(key), Value(value) @@ -300,6 +499,16 @@ void TFixture::SetUp(NUnitTest::TTestContext&) Setup = std::make_unique(TEST_CASE_NAME, settings); Driver = std::make_unique(Setup->MakeDriver()); + auto tableSettings = NTable::TClientSettings().SessionPoolSettings(NTable::TSessionPoolSettings() + .MaxActiveSessions(3000) + ); + + auto querySettings = NQuery::TClientSettings().SessionPoolSettings(NQuery::TSessionPoolSettings() + .MaxActiveSessions(3000) + ); + + TableClient = std::make_unique(*Driver, tableSettings); + QueryClient = std::make_unique(*Driver, querySettings); } void TFixture::NotifySchemeShard(const TFeatureFlags& flags) @@ -317,30 +526,154 @@ void TFixture::NotifySchemeShard(const TFeatureFlags& flags) runtime.GrabEdgeEvent(); } -NTable::TSession TFixture::CreateTableSession() +TFixture::TTableSession::TTableSession(NTable::TTableClient& client) + : Session_(Init(client)) +{ +} + +NTable::TSession TFixture::TTableSession::Init(NTable::TTableClient& client) +{ + auto result = client.GetSession().ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + return result.GetSession(); +} + +std::vector TFixture::TTableSession::Execute(const std::string& query, + TTransactionBase* tx, + bool commit, + const TParams& params) +{ + auto txTable = dynamic_cast(tx); + auto txControl = NTable::TTxControl::Tx(*txTable).CommitTx(commit); + + auto result = Session_.ExecuteDataQuery(query, txControl, params).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + return std::move(result).ExtractResultSets(); +} + +TFixture::ISession::TExecuteInTxResult TFixture::TTableSession::ExecuteInTx(const std::string& query, + bool commit, + const TParams& params) +{ + auto txControl = NTable::TTxControl::BeginTx().CommitTx(commit); + + auto result = Session_.ExecuteDataQuery(query, txControl, params).GetValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + return {std::move(result).ExtractResultSets(), std::make_unique(*result.GetTransaction())}; +} + +std::unique_ptr TFixture::TTableSession::BeginTx() +{ + while (true) { + auto result = Session_.BeginTransaction().ExtractValueSync(); + if (result.GetStatus() != EStatus::SESSION_BUSY) { + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + return std::make_unique(result.GetTransaction()); + } + Sleep(TDuration::MilliSeconds(100)); + } +} + +void TFixture::TTableSession::CommitTx(TTransactionBase& tx, EStatus status) +{ + auto txTable = dynamic_cast(tx); + while (true) { + auto result = txTable.Commit().ExtractValueSync(); + if (result.GetStatus() != EStatus::SESSION_BUSY) { + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, result.GetIssues().ToString()); + return; + } + Sleep(TDuration::MilliSeconds(100)); + } +} + +void TFixture::TTableSession::RollbackTx(TTransactionBase& tx, EStatus status) +{ + auto txTable = dynamic_cast(tx); + while (true) { + auto result = txTable.Rollback().ExtractValueSync(); + if (result.GetStatus() != EStatus::SESSION_BUSY) { + UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, result.GetIssues().ToString()); + return; + } + Sleep(TDuration::MilliSeconds(100)); + } +} + +void TFixture::TTableSession::Close() +{ + Session_.Close(); +} + +TAsyncStatus TFixture::TTableSession::AsyncCommitTx(TTransactionBase& tx) +{ + auto txTable = dynamic_cast(tx); + return txTable.Commit().Apply([](auto result) { + return TStatus(result.GetValue()); + }); +} + +TFixture::TQuerySession::TQuerySession(NQuery::TQueryClient& client, + const std::string& endpoint, + const std::string& database) + : Session_(Init(client)) + , Endpoint_(endpoint) + , Database_(database) +{ +} + +NQuery::TSession TFixture::TQuerySession::Init(NQuery::TQueryClient& client) { - NTable::TTableClient client(GetDriver()); - auto result = client.CreateSession().ExtractValueSync(); + auto result = client.GetSession().ExtractValueSync(); UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); return result.GetSession(); } -NTable::TTransaction TFixture::BeginTx(NTable::TSession& session) +std::vector TFixture::TQuerySession::Execute(const std::string& query, + TTransactionBase* tx, + bool commit, + const TParams& params) +{ + auto txQuery = dynamic_cast(tx); + auto txControl = NQuery::TTxControl::Tx(*txQuery).CommitTx(commit); + + auto result = Session_.ExecuteQuery(query, txControl, params).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + return result.GetResultSets(); +} + +TFixture::ISession::TExecuteInTxResult TFixture::TQuerySession::ExecuteInTx(const std::string& query, + bool commit, + const TParams& params) +{ + auto txControl = NQuery::TTxControl::BeginTx().CommitTx(commit); + + auto result = Session_.ExecuteQuery(query, txControl, params).ExtractValueSync(); + UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + return {result.GetResultSets(), std::make_unique(*result.GetTransaction())}; +} + +std::unique_ptr TFixture::TQuerySession::BeginTx() { while (true) { - auto result = session.BeginTransaction().ExtractValueSync(); + auto result = Session_.BeginTransaction(NQuery::TTxSettings()).ExtractValueSync(); if (result.GetStatus() != EStatus::SESSION_BUSY) { UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); - return result.GetTransaction(); + return std::make_unique(result.GetTransaction()); } Sleep(TDuration::MilliSeconds(100)); } } -void TFixture::CommitTx(NTable::TTransaction& tx, EStatus status) +void TFixture::TQuerySession::CommitTx(TTransactionBase& tx, EStatus status) { + auto txQuery = dynamic_cast(tx); while (true) { - auto result = tx.Commit().ExtractValueSync(); + auto result = txQuery.Commit().ExtractValueSync(); if (result.GetStatus() != EStatus::SESSION_BUSY) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, result.GetIssues().ToString()); return; @@ -349,10 +682,11 @@ void TFixture::CommitTx(NTable::TTransaction& tx, EStatus status) } } -void TFixture::RollbackTx(NTable::TTransaction& tx, EStatus status) +void TFixture::TQuerySession::RollbackTx(TTransactionBase& tx, EStatus status) { + auto txQuery = dynamic_cast(tx); while (true) { - auto result = tx.Rollback().ExtractValueSync(); + auto result = txQuery.Rollback().ExtractValueSync(); if (result.GetStatus() != EStatus::SESSION_BUSY) { UNIT_ASSERT_VALUES_EQUAL_C(result.GetStatus(), status, result.GetIssues().ToString()); return; @@ -361,6 +695,57 @@ void TFixture::RollbackTx(NTable::TTransaction& tx, EStatus status) } } +void TFixture::TQuerySession::Close() +{ + // SDK doesn't provide a method to close the session for Query Client, so we use grpc API directly + auto credentials = grpc::InsecureChannelCredentials(); + auto channel = grpc::CreateChannel(TString(Endpoint_), credentials); + auto stub = Ydb::Query::V1::QueryService::NewStub(channel); + + grpc::ClientContext context; + context.AddMetadata("x-ydb-database", TString(Database_)); + + Ydb::Query::DeleteSessionRequest request; + request.set_session_id(Session_.GetId()); + + Ydb::Query::DeleteSessionResponse response; + auto status = stub->DeleteSession(&context, request, &response); + + NIssue::TIssues issues; + NYdb::NIssue::IssuesFromMessage(response.issues(), issues); + UNIT_ASSERT_C(status.ok(), status.error_message()); + UNIT_ASSERT_VALUES_EQUAL_C(response.status(), Ydb::StatusIds::SUCCESS, issues.ToString()); +} + +TAsyncStatus TFixture::TQuerySession::AsyncCommitTx(TTransactionBase& tx) +{ + auto txQuery = dynamic_cast(tx); + return txQuery.Commit().Apply([](auto result) { + return TStatus(result.GetValue()); + }); +} + +std::unique_ptr TFixture::CreateSession() +{ + switch (GetClientType()) { + case EClientType::Table: { + UNIT_ASSERT_C(TableClient, "TableClient is not initialized"); + return std::make_unique(*TableClient); + } + case EClientType::Query: { + UNIT_ASSERT_C(QueryClient, "QueryClient is not initialized"); + return std::make_unique(*QueryClient, + Setup->GetEndpoint(), + Setup->GetDatabase()); + } + case EClientType::None: { + UNIT_FAIL("CreateSession is forbidden for None client type"); + } + } + + return nullptr; +} + auto TFixture::CreateReader() -> TTopicReadSessionPtr { NTopic::TTopicClient client(GetDriver()); @@ -370,7 +755,7 @@ auto TFixture::CreateReader() -> TTopicReadSessionPtr return client.CreateReadSession(options); } -void TFixture::StartPartitionSession(TTopicReadSessionPtr reader, NTable::TTransaction& tx, ui64 offset) +void TFixture::StartPartitionSession(TTopicReadSessionPtr reader, TTransactionBase& tx, ui64 offset) { auto event = ReadEvent(reader, tx); UNIT_ASSERT_VALUES_EQUAL(event.GetCommittedOffset(), offset); @@ -384,7 +769,7 @@ void TFixture::StartPartitionSession(TTopicReadSessionPtr reader, ui64 offset) event.Confirm(); } -void TFixture::ReadMessage(TTopicReadSessionPtr reader, NTable::TTransaction& tx, ui64 offset) +void TFixture::ReadMessage(TTopicReadSessionPtr reader, TTransactionBase& tx, ui64 offset) { TReadMessageSettings settings { .Tx = tx, @@ -406,7 +791,7 @@ void TFixture::ReadMessage(TTopicReadSessionPtr reader, const TReadMessageSettin } template -E TFixture::ReadEvent(TTopicReadSessionPtr reader, NTable::TTransaction& tx) +E TFixture::ReadEvent(TTopicReadSessionPtr reader, TTransactionBase& tx) { NTopic::TReadSessionGetEventSettings options; options.Block(true); @@ -448,7 +833,7 @@ void TFixture::WriteMessage(const TString& message) void TFixture::WriteMessages(const TVector& messages, const TString& topic, const TString& groupId, - NTable::TTransaction& tx) + TTransactionBase& tx) { NTopic::TWriteSessionSettings options; options.Path(topic); @@ -539,10 +924,15 @@ const TDriver& TFixture::GetDriver() const return *Driver; } +NTable::TTableClient& TFixture::GetTableClient() +{ + return *TableClient; +} + void TFixture::WriteToTopicWithInvalidTxId(bool invalidTxId) { - auto tableSession = CreateTableSession(); - auto tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); NTopic::TWriteSessionSettings options; options.Path(TEST_TOPIC); @@ -556,13 +946,12 @@ void TFixture::WriteToTopicWithInvalidTxId(bool invalidTxId) auto token = std::move(std::get(event.value()).ContinuationToken); NTopic::TWriteMessage params("message"); - params.Tx(tx); + params.Tx(*tx); if (invalidTxId) { - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); } else { - auto result = tableSession.Close().ExtractValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + session->Close(); } writeSession->Write(std::move(token), std::move(params)); @@ -581,32 +970,32 @@ void TFixture::WriteToTopicWithInvalidTxId(bool invalidTxId) } } -Y_UNIT_TEST_F(SessionAbort, TFixture) +void TFixture::TestSessionAbort() { { auto reader = CreateReader(); - auto session = CreateTableSession(); - auto tx = BeginTx(session); + auto session = CreateSession(); + auto tx = session->BeginTx(); - StartPartitionSession(reader, tx, 0); + StartPartitionSession(reader, *tx, 0); WriteMessage("message #0"); - ReadMessage(reader, tx, 0); + ReadMessage(reader, *tx, 0); WriteMessage("message #1"); - ReadMessage(reader, tx, 1); + ReadMessage(reader, *tx, 1); } { - auto session = CreateTableSession(); - auto tx = BeginTx(session); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto reader = CreateReader(); - StartPartitionSession(reader, tx, 0); + StartPartitionSession(reader, *tx, 0); - ReadMessage(reader, tx, 0); + ReadMessage(reader, *tx, 0); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); } { @@ -616,46 +1005,81 @@ Y_UNIT_TEST_F(SessionAbort, TFixture) } } -Y_UNIT_TEST_F(TwoSessionOneConsumer, TFixture) +Y_UNIT_TEST_F(SessionAbort_Table, TFixtureTable) +{ + TestSessionAbort(); +} + +Y_UNIT_TEST_F(SessionAbort_Query, TFixtureQuery) +{ + TestSessionAbort(); +} + +void TFixture::TestTwoSessionOneConsumer() { WriteMessage("message #0"); - auto session1 = CreateTableSession(); - auto tx1 = BeginTx(session1); + auto session1 = CreateSession(); + auto tx1 = session1->BeginTx(); { auto reader = CreateReader(); - StartPartitionSession(reader, tx1, 0); - ReadMessage(reader, tx1, 0); + StartPartitionSession(reader, *tx1, 0); + ReadMessage(reader, *tx1, 0); } - auto session2 = CreateTableSession(); - auto tx2 = BeginTx(session2); + auto session2 = CreateSession(); + auto tx2 = session2->BeginTx(); { auto reader = CreateReader(); - StartPartitionSession(reader, tx2, 0); - ReadMessage(reader, tx2, 0); + StartPartitionSession(reader, *tx2, 0); + ReadMessage(reader, *tx2, 0); } - CommitTx(tx2, EStatus::SUCCESS); - CommitTx(tx1, EStatus::ABORTED); + session2->CommitTx(*tx2, EStatus::SUCCESS); + session1->CommitTx(*tx1, EStatus::ABORTED); +} + +Y_UNIT_TEST_F(TwoSessionOneConsumer_Table, TFixtureTable) +{ + TestTwoSessionOneConsumer(); +} + +Y_UNIT_TEST_F(TwoSessionOneConsumer_Query, TFixtureQuery) +{ + TestTwoSessionOneConsumer(); } -Y_UNIT_TEST_F(Offsets_Cannot_Be_Promoted_When_Reading_In_A_Transaction, TFixture) +void TFixture::TestOffsetsCannotBePromotedWhenReadingInATransaction() { WriteMessage("message"); - auto session = CreateTableSession(); - auto tx = BeginTx(session); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto reader = CreateReader(); - StartPartitionSession(reader, tx, 0); + StartPartitionSession(reader, *tx, 0); + + UNIT_ASSERT_EXCEPTION(ReadMessage(reader, {.Tx = *tx, .CommitOffsets = true}), yexception); +} + +Y_UNIT_TEST_F(Offsets_Cannot_Be_Promoted_When_Reading_In_A_Transaction_Table, TFixtureTable) +{ + TestOffsetsCannotBePromotedWhenReadingInATransaction(); +} + +Y_UNIT_TEST_F(Offsets_Cannot_Be_Promoted_When_Reading_In_A_Transaction_Query, TFixtureQuery) +{ + TestOffsetsCannotBePromotedWhenReadingInATransaction(); +} - UNIT_ASSERT_EXCEPTION(ReadMessage(reader, {.Tx = tx, .CommitOffsets = true}), yexception); +Y_UNIT_TEST_F(WriteToTopic_Invalid_Session_Table, TFixtureTable) +{ + WriteToTopicWithInvalidTxId(false); } -Y_UNIT_TEST_F(WriteToTopic_Invalid_Session, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Invalid_Session_Query, TFixtureQuery) { WriteToTopicWithInvalidTxId(false); } @@ -665,7 +1089,7 @@ Y_UNIT_TEST_F(WriteToTopic_Invalid_Session, TFixture) // WriteToTopicWithInvalidTxId(true); //} -Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) +void TFixture::TestWriteToTopicTwoWriteSession() { TString topicPath[2] = { TString{TEST_TOPIC}, @@ -684,7 +1108,7 @@ Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) auto writeMessage = [](auto& ws, const TString& message, auto& tx) { NTopic::TWriteMessage params(message); - params.Tx(tx); + params.Tx(*tx); auto event = ws->GetEvent(true); UNIT_ASSERT(event && std::holds_alternative(event.value())); @@ -693,8 +1117,8 @@ Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) ws->Write(std::move(token), std::move(params)); }; - auto tableSession = CreateTableSession(); - auto tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); NTopic::TTopicClient client(GetDriver()); @@ -729,6 +1153,16 @@ Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession, TFixture) UNIT_ASSERT_VALUES_EQUAL(acks, 2); } +Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession_Table, TFixtureTable) +{ + TestWriteToTopicTwoWriteSession(); +} + +Y_UNIT_TEST_F(WriteToTopic_Two_WriteSession_Query, TFixtureQuery) +{ + TestWriteToTopicTwoWriteSession(); +} + auto TFixture::CreateTopicWriteSession(const TString& topicPath, const TString& messageGroupId, std::optional partitionId) -> TTopicWriteSessionPtr @@ -813,7 +1247,7 @@ auto TFixture::GetTopicReadSession(const TString& topicPath, void TFixture::TTopicWriteSessionContext::WaitForContinuationToken() { - while (!ContinuationToken.Defined()) { + while (!ContinuationToken.has_value()) { WaitForEvent(); } } @@ -837,13 +1271,13 @@ void TFixture::TTopicWriteSessionContext::WaitForEvent() break; } } - } else if (auto* e = std::get_if(&event)) { + } else if ([[maybe_unused]] auto* e = std::get_if(&event)) { UNIT_FAIL(""); } } } -void TFixture::TTopicWriteSessionContext::Write(const TString& message, NTable::TTransaction* tx) +void TFixture::TTopicWriteSessionContext::Write(const std::string& message, TTransactionBase* tx) { NTopic::TWriteMessage params(message); @@ -855,7 +1289,7 @@ void TFixture::TTopicWriteSessionContext::Write(const TString& message, NTable:: std::move(params)); ++WriteCount; - ContinuationToken = Nothing(); + ContinuationToken = std::nullopt; } void TFixture::CloseTopicWriteSession(const TString& topicPath, @@ -883,19 +1317,19 @@ void TFixture::CloseTopicReadSession(const TString& topicPath, void TFixture::WriteToTopic(const TString& topicPath, const TString& messageGroupId, const TString& message, - NTable::TTransaction* tx, + TTransactionBase* tx, std::optional partitionId) { TTopicWriteSessionContext& context = GetTopicWriteSession(topicPath, messageGroupId, partitionId); context.WaitForContinuationToken(); - UNIT_ASSERT(context.ContinuationToken.Defined()); + UNIT_ASSERT(context.ContinuationToken.has_value()); context.Write(message, tx); } TVector TFixture::ReadFromTopic(const TString& topicPath, const TString& consumerName, const TDuration& duration, - NTable::TTransaction* tx, + TTransactionBase* tx, TMaybe partitionId) { TVector messages; @@ -1180,19 +1614,19 @@ void TFixture::TestWriteToTopic1() CreateTopic("topic_A"); CreateTopic("topic_B"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #4", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #4", tx.get()); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #5", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #6", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #7", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #8", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #9", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #5", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #6", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #7", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #8", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #9", tx.get()); { auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); @@ -1204,7 +1638,7 @@ void TFixture::TestWriteToTopic1() UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); { auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 4); @@ -1224,16 +1658,16 @@ void TFixture::TestWriteToTopic4() CreateTopic("topic_A"); CreateTopic("topic_B"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx_1 = BeginTx(tableSession); + auto session = CreateSession(); + auto tx_1 = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx_1); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx_1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx_1.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", tx_1.get()); - NTable::TTransaction tx_2 = BeginTx(tableSession); + auto tx_2 = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", &tx_2); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #4", &tx_2); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", tx_2.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #4", tx_2.get()); auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); @@ -1241,8 +1675,8 @@ void TFixture::TestWriteToTopic4() messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2)); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); - CommitTx(tx_2, EStatus::SUCCESS); - CommitTx(tx_1, EStatus::ABORTED); + session->CommitTx(*tx_2, EStatus::SUCCESS); + session->CommitTx(*tx_1, EStatus::ABORTED); messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); @@ -1257,17 +1691,17 @@ void TFixture::TestWriteToTopic7() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #2", tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #3"); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #4"); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #5", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #6", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #5", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #6", tx.get()); { auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 2); @@ -1275,7 +1709,7 @@ void TFixture::TestWriteToTopic7() UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #4"); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); { auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 4); @@ -1288,22 +1722,22 @@ void TFixture::TestWriteToTopic9() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx_1 = BeginTx(tableSession); + auto session = CreateSession(); + auto tx_1 = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx_1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx_1.get()); - NTable::TTransaction tx_2 = BeginTx(tableSession); + auto tx_2 = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx_2); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx_2.get()); { auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); } - CommitTx(tx_2, EStatus::SUCCESS); - CommitTx(tx_1, EStatus::ABORTED); + session->CommitTx(*tx_2, EStatus::SUCCESS); + session->CommitTx(*tx_1, EStatus::ABORTED); { auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); @@ -1316,22 +1750,22 @@ void TFixture::TestWriteToTopic10() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); + auto session = CreateSession(); { - NTable::TTransaction tx_1 = BeginTx(tableSession); + auto tx_1 = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx_1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx_1.get()); - CommitTx(tx_1, EStatus::SUCCESS); + session->CommitTx(*tx_1, EStatus::SUCCESS); } { - NTable::TTransaction tx_2 = BeginTx(tableSession); + auto tx_2 = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx_2); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx_2.get()); - CommitTx(tx_2, EStatus::SUCCESS); + session->CommitTx(*tx_2, EStatus::SUCCESS); } { @@ -1357,14 +1791,14 @@ void TFixture::TestWriteToTopic24() CreateTopic("topic_A"); CreateTable("/Root/table_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx); + WriteToTable("table_A", records, *session, tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); @@ -1390,17 +1824,17 @@ void TFixture::TestWriteToTopic26() WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", nullptr, PARTITION_0); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", nullptr, PARTITION_0); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), &tx, PARTITION_0); + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), tx.get(), PARTITION_0); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 3); for (const auto& m : messages) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, m, &tx, PARTITION_1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, m, tx.get(), PARTITION_1); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), nullptr, PARTITION_1); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 3); @@ -1416,20 +1850,20 @@ void TFixture::TestWriteToTopic27() WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", nullptr, 0); WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", nullptr, 0); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), &tx, 0); + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), tx.get(), 0); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); - WriteToTopic("topic_C", TEST_MESSAGE_GROUP_ID, messages[0], &tx, 0); + WriteToTopic("topic_C", TEST_MESSAGE_GROUP_ID, messages[0], tx.get(), 0); - messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2), &tx, 0); + messages = ReadFromTopic("topic_B", TEST_CONSUMER, TDuration::Seconds(2), tx.get(), 0); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); - WriteToTopic("topic_C", TEST_MESSAGE_GROUP_ID, messages[0], &tx, 0); + WriteToTopic("topic_C", TEST_MESSAGE_GROUP_ID, messages[0], tx.get(), 0); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); messages = ReadFromTopic("topic_C", TEST_CONSUMER, TDuration::Seconds(2), nullptr, 0); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); @@ -1488,30 +1922,35 @@ bool TFixture::GetAllowOlapDataQuery() const return false; } -Y_UNIT_TEST_F(WriteToTopic_Demo_1, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_1_Table, TFixtureTable) +{ + TestWriteToTopic1(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_1_Query, TFixtureQuery) { TestWriteToTopic1(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_2, TFixture) +void TFixture::TestWriteToTopic2() { CreateTopic("topic_A"); CreateTopic("topic_B"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #2", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #3", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #4", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #2", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #3", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #4", tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #5"); WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_2, "message #6"); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #7", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #8", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #9", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #7", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #8", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_1, "message #9", tx.get()); { auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); @@ -1525,7 +1964,7 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_2, TFixture) UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #6"); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); { auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 4); @@ -1540,16 +1979,26 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_2, TFixture) } } -Y_UNIT_TEST_F(WriteToTopic_Demo_3, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_2_Table, TFixtureTable) +{ + TestWriteToTopic2(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_2_Query, TFixtureQuery) +{ + TestWriteToTopic2(); +} + +void TFixture::TestWriteToTopic3() { CreateTopic("topic_A"); CreateTopic("topic_B"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3"); @@ -1557,14 +2006,14 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_3, TFixture) UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #3"); - CommitTx(tx, EStatus::ABORTED); + session->CommitTx(*tx, EStatus::ABORTED); - tx = BeginTx(tableSession); + tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 1); @@ -1575,34 +2024,49 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_3, TFixture) UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #2"); } -Y_UNIT_TEST_F(WriteToTopic_Demo_4, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_3_Table, TFixtureTable) +{ + TestWriteToTopic3(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_3_Query, TFixtureQuery) +{ + TestWriteToTopic3(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_4_Table, TFixtureTable) +{ + TestWriteToTopic4(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_4_Query, TFixtureQuery) { TestWriteToTopic4(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_5, TFixture) +void TFixture::TestWriteToTopic5() { CreateTopic("topic_A"); CreateTopic("topic_B"); - NTable::TSession tableSession = CreateTableSession(); + auto session = CreateSession(); { - NTable::TTransaction tx_1 = BeginTx(tableSession); + auto tx_1 = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx_1); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx_1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx_1.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", tx_1.get()); - CommitTx(tx_1, EStatus::SUCCESS); + session->CommitTx(*tx_1, EStatus::SUCCESS); } { - NTable::TTransaction tx_2 = BeginTx(tableSession); + auto tx_2 = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", &tx_2); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #4", &tx_2); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3", tx_2.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #4", tx_2.get()); - CommitTx(tx_2, EStatus::SUCCESS); + session->CommitTx(*tx_2, EStatus::SUCCESS); } { @@ -1618,22 +2082,32 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_5, TFixture) } } -Y_UNIT_TEST_F(WriteToTopic_Demo_6, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_5_Table, TFixtureTable) +{ + TestWriteToTopic5(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_5_Query, TFixtureQuery) +{ + TestWriteToTopic5(); +} + +void TFixture::TestWriteToTopic6() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); { auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); { auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 2); @@ -1644,19 +2118,34 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_6, TFixture) DescribeTopic("topic_A"); } -Y_UNIT_TEST_F(WriteToTopic_Demo_7, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_6_Table, TFixtureTable) +{ + TestWriteToTopic6(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_6_Query, TFixtureQuery) +{ + TestWriteToTopic6(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_7_Table, TFixtureTable) +{ + TestWriteToTopic7(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_7_Query, TFixtureQuery) { TestWriteToTopic7(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_8, TFixture) +void TFixture::TestWriteToTopic8() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2"); @@ -1666,13 +2155,13 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_8, TFixture) UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #2"); } - CommitTx(tx, EStatus::ABORTED); + session->CommitTx(*tx, EStatus::ABORTED); - tx = BeginTx(tableSession); + tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); { auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2)); @@ -1681,24 +2170,44 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_8, TFixture) } } -Y_UNIT_TEST_F(WriteToTopic_Demo_9, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_8_Table, TFixtureTable) { - TestWriteToTopic9(); + TestWriteToTopic8(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_10, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_8_Query, TFixtureQuery) { - TestWriteToTopic10(); + TestWriteToTopic8(); } -NPQ::TWriteId TFixture::GetTransactionWriteId(const TActorId& actorId, - ui64 tabletId) +Y_UNIT_TEST_F(WriteToTopic_Demo_9_Table, TFixtureTable) { - using TEvKeyValue = NKikimr::TEvKeyValue; + TestWriteToTopic9(); +} - auto request = std::make_unique(); - request->Record.SetCookie(12345); - request->Record.AddCmdRead()->SetKey("_txinfo"); +Y_UNIT_TEST_F(WriteToTopic_Demo_9_Query, TFixtureQuery) +{ + TestWriteToTopic9(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_10_Table, TFixtureTable) +{ + TestWriteToTopic10(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_10_Query, TFixtureQuery) +{ + TestWriteToTopic10(); +} + +NPQ::TWriteId TFixture::GetTransactionWriteId(const TActorId& actorId, + ui64 tabletId) +{ + using TEvKeyValue = NKikimr::TEvKeyValue; + + auto request = std::make_unique(); + request->Record.SetCookie(12345); + request->Record.AddCmdRead()->SetKey("_txinfo"); auto& runtime = Setup->GetRuntime(); @@ -1913,21 +2422,21 @@ void TFixture::TestTheCompletionOfATransaction(const TTransactionCompletionTestD } { - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); for (auto& topic : d.Topics) { - WriteToTopic(topic, TEST_MESSAGE_GROUP_ID, "message", &tx); + WriteToTopic(topic, TEST_MESSAGE_GROUP_ID, "message", tx.get()); // TODO: нужен callback для RollbakTx WaitForAcks(topic, TEST_MESSAGE_GROUP_ID); } switch (d.EndOfTransaction) { case Commit: - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); break; case Rollback: - RollbackTx(tx, EStatus::SUCCESS); + session->RollbackTx(*tx, EStatus::SUCCESS); break; case CloseTableSession: break; @@ -1945,114 +2454,162 @@ void TFixture::TestTheCompletionOfATransaction(const TTransactionCompletionTestD } } -NTable::TDataQueryResult TFixture::ExecuteDataQuery(NTable::TSession session, const TString& query, const NTable::TTxControl& control) +Y_UNIT_TEST_F(WriteToTopic_Demo_11_Table, TFixtureTable) { - auto status = session.ExecuteDataQuery(query, control).GetValueSync(); - UNIT_ASSERT_C(status.IsSuccess(), status.GetIssues().ToString()); - return status; + TestWriteToTopic11(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_11, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_11_Query, TFixtureQuery) { TestWriteToTopic11(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_12, TFixture) +void TFixture::TestWriteToTopic12() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); DeleteSupportivePartition("topic_A", 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); WaitForSessionClose("topic_A", TEST_MESSAGE_GROUP_ID, NYdb::EStatus::PRECONDITION_FAILED); } -Y_UNIT_TEST_F(WriteToTopic_Demo_13, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_12_Table, TFixtureTable) +{ + TestWriteToTopic12(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_12_Query, TFixtureQuery) +{ + TestWriteToTopic12(); +} + +void TFixture::TestWriteToTopic13() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message", tx.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); DeleteSupportivePartition("topic_A", 0); - CommitTx(tx, EStatus::ABORTED); + session->CommitTx(*tx, EStatus::ABORTED); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_13_Table, TFixtureTable) +{ + TestWriteToTopic13(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_14, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_13_Query, TFixtureQuery) +{ + TestWriteToTopic13(); +} + +void TFixture::TestWriteToTopic14() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); DeleteSupportivePartition("topic_A", 0); CloseTopicWriteSession("topic_A", TEST_MESSAGE_GROUP_ID); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); + + session->CommitTx(*tx, EStatus::ABORTED); +} - CommitTx(tx, EStatus::ABORTED); +Y_UNIT_TEST_F(WriteToTopic_Demo_14_Table, TFixtureTable) +{ + TestWriteToTopic14(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_14_Query, TFixtureQuery) +{ + TestWriteToTopic14(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_15, TFixture) +void TFixture::TestWriteToTopic15() { // the session of writing to the topic can be closed before the commit CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, "message #1", tx.get()); CloseTopicWriteSession("topic_A", TEST_MESSAGE_GROUP_ID_1); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, "message #2", tx.get()); CloseTopicWriteSession("topic_A", TEST_MESSAGE_GROUP_ID_2); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 2); UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #2"); } -Y_UNIT_TEST_F(WriteToTopic_Demo_16, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_15_Table, TFixtureTable) +{ + TestWriteToTopic15(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_15_Query, TFixtureQuery) +{ + TestWriteToTopic15(); +} + +void TFixture::TestWriteToTopic16() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); RestartPQTablet("topic_A", 0); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 2); UNIT_ASSERT_VALUES_EQUAL(messages[0], "message #1"); UNIT_ASSERT_VALUES_EQUAL(messages[1], "message #2"); } -Y_UNIT_TEST_F(WriteToTopic_Demo_17, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_16_Table, TFixtureTable) +{ + TestWriteToTopic16(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_16_Query, TFixtureQuery) +{ + TestWriteToTopic16(); +} + +void TFixture::TestWriteToTopic17() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(22'000'000, 'x')); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(100, 'x')); @@ -2060,11 +2617,11 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_17, TFixture) WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(300, 'x')); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(10'000'000, 'x')); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString( 6'000'000, 'x'), &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(20'000'000, 'x'), &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString( 7'000'000, 'x'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString( 6'000'000, 'x'), tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(20'000'000, 'x'), tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString( 7'000'000, 'x'), tx.get()); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); //RestartPQTablet("topic_A", 0); @@ -2079,6 +2636,16 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_17, TFixture) UNIT_ASSERT_VALUES_EQUAL(messages[7].size(), 7'000'000); } +Y_UNIT_TEST_F(WriteToTopic_Demo_17_Table, TFixtureTable) +{ + TestWriteToTopic17(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_17_Query, TFixtureQuery) +{ + TestWriteToTopic17(); +} + void TFixture::TestTxWithBigBlobs(const TTestTxWithBigBlobsParams& params) { size_t oldHeadMsgCount = 0; @@ -2087,8 +2654,8 @@ void TFixture::TestTxWithBigBlobs(const TTestTxWithBigBlobsParams& params) CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); for (size_t i = 0; i < params.OldHeadCount; ++i) { WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(100'000, 'x')); @@ -2096,12 +2663,12 @@ void TFixture::TestTxWithBigBlobs(const TTestTxWithBigBlobsParams& params) } for (size_t i = 0; i < params.BigBlobsCount; ++i) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(7'000'000, 'x'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(7'000'000, 'x'), tx.get()); ++bigBlobMsgCount; } for (size_t i = 0; i < params.NewHeadCount; ++i) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(100'000, 'x'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(100'000, 'x'), tx.get()); ++newHeadMsgCount; } @@ -2109,7 +2676,7 @@ void TFixture::TestTxWithBigBlobs(const TTestTxWithBigBlobsParams& params) RestartPQTablet("topic_A", 0); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); if (params.RestartMode == ERestartAfterCommit) { RestartPQTablet("topic_A", 0); @@ -2143,13 +2710,22 @@ void TFixture::TestTxWithBigBlobs(const TTestTxWithBigBlobsParams& params) } #define Y_UNIT_TEST_WITH_REBOOTS(name, oldHeadCount, bigBlobsCount, newHeadCount) \ -Y_UNIT_TEST_F(name##_RestartNo, TFixture) { \ +Y_UNIT_TEST_F(name##_RestartNo_Table, TFixtureTable) { \ TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartNo}); \ } \ -Y_UNIT_TEST_F(name##_RestartBeforeCommit, TFixture) { \ +Y_UNIT_TEST_F(name##_RestartNo_Query, TFixtureQuery) { \ + TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartNo}); \ +} \ +Y_UNIT_TEST_F(name##_RestartBeforeCommit_Table, TFixtureTable) { \ + TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartBeforeCommit}); \ +} \ +Y_UNIT_TEST_F(name##_RestartBeforeCommit_Query, TFixtureQuery) { \ TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartBeforeCommit}); \ } \ -Y_UNIT_TEST_F(name##_RestartAfterCommit, TFixture) { \ +Y_UNIT_TEST_F(name##_RestartAfterCommit_Table, TFixtureTable) { \ + TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartAfterCommit}); \ +} \ +Y_UNIT_TEST_F(name##_RestartAfterCommit_Query, TFixtureQuery) { \ TestTxWithBigBlobs({.OldHeadCount = oldHeadCount, .BigBlobsCount = bigBlobsCount, .NewHeadCount = newHeadCount, .RestartMode = ERestartAfterCommit}); \ } @@ -2167,7 +2743,10 @@ void TFixture::CreateTable(const TString& tablePath) TString path = (tablePath[0] != '/') ? ("/Root/" + tablePath) : tablePath; - NTable::TSession session = CreateTableSession(); + auto createSessionResult = GetTableClient().CreateSession().ExtractValueSync(); + UNIT_ASSERT_C(createSessionResult.IsSuccess(), createSessionResult.GetIssues().ToString()); + auto session = createSessionResult.GetSession(); + auto desc = NTable::TTableBuilder() .AddNonNullableColumn("key", EPrimitiveType::Utf8) .AddNonNullableColumn("value", EPrimitiveType::Utf8) @@ -2213,23 +2792,21 @@ auto TFixture::MakeJsonDoc(const TVector& records) -> TString void TFixture::WriteToTable(const TString& tablePath, const TVector& records, - NTable::TTransaction* tx) + ISession& session, + TTransactionBase* tx) { TString query = Sprintf("DECLARE $key AS Utf8;" "DECLARE $value AS Utf8;" "UPSERT INTO `%s` (key, value) VALUES ($key, $value);", tablePath.data()); - NTable::TSession session = tx->GetSession(); for (const auto& r : records) { - auto params = session.GetParamsBuilder() - .AddParam("$key").Utf8(r.Key).Build() - .AddParam("$value").Utf8(r.Value).Build() + auto params = TParamsBuilder() + .AddParam("$key").Utf8(r.Key).Build() + .AddParam("$value").Utf8(r.Value).Build() .Build(); - auto result = session.ExecuteDataQuery(query, - NYdb::NTable::TTxControl::Tx(*tx), - params).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + + session.Execute(query, tx, false, params); } } @@ -2237,25 +2814,28 @@ size_t TFixture::GetTableRecordsCount(const TString& tablePath) { TString query = Sprintf(R"(SELECT COUNT(*) FROM `%s`)", tablePath.data()); - NTable::TSession session = CreateTableSession(); - NTable::TTransaction tx = BeginTx(session); + auto session = CreateSession(); + auto tx = session->BeginTx(); - auto result = session.ExecuteDataQuery(query, - NYdb::NTable::TTxControl::Tx(tx).CommitTx(true)).GetValueSync(); - UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString()); + auto result = session->Execute(query, tx.get()); - NYdb::TResultSetParser parser(result.GetResultSet(0)); + NYdb::TResultSetParser parser(result.at(0)); UNIT_ASSERT(parser.TryNextRow()); return parser.ColumnParser(0).GetUint64(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_24, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_24_Table, TFixtureTable) +{ + TestWriteToTopic24(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_24_Query, TFixtureQuery) { TestWriteToTopic24(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_25, TFixture) +void TFixture::TestWriteToTopic25() { // // the test verifies a transaction in which data is read from one topic and written to another @@ -2267,45 +2847,65 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_25, TFixture) WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2"); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #3"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), &tx); + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), tx.get()); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 3); for (const auto& m : messages) { - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, m, &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, m, tx.get()); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); Read_Exactly_N_Messages_From_Topic("topic_B", TEST_CONSUMER, 3); } -Y_UNIT_TEST_F(WriteToTopic_Demo_26, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_25_Table, TFixtureTable) +{ + TestWriteToTopic25(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_25_Query, TFixtureQuery) +{ + TestWriteToTopic25(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_26_Table, TFixtureTable) +{ + TestWriteToTopic26(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_26_Query, TFixtureQuery) { TestWriteToTopic26(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_27, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_27_Table, TFixtureTable) { TestWriteToTopic27(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_28, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_27_Query, TFixtureQuery) +{ + TestWriteToTopic27(); +} + +void TFixture::TestWriteToTopic28() { // The test verifies that the `WriteInflightSize` is correctly considered for the main partition. // Writing to the service partition does not change the `WriteInflightSize` of the main one. CreateTopic("topic_A", TEST_CONSUMER); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); TString message(16'000, 'a'); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, TString(16'000, 'a'), &tx, 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, TString(16'000, 'a'), tx.get(), 0); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, TString(20'000, 'b'), nullptr, 0); @@ -2313,80 +2913,179 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_28, TFixture) UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); } +Y_UNIT_TEST_F(WriteToTopic_Demo_28_Table, TFixtureTable) +{ + TestWriteToTopic28(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_28_Query, TFixtureQuery) +{ + TestWriteToTopic28(); +} + void TFixture::WriteMessagesInTx(size_t big, size_t small) { CreateTopic("topic_A", TEST_CONSUMER); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); for (size_t i = 0; i < big; ++i) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(7'000'000, 'x'), &tx, 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(7'000'000, 'x'), tx.get(), 0); } for (size_t i = 0; i < small; ++i) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(16'384, 'x'), &tx, 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(16'384, 'x'), tx.get(), 0); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); } -Y_UNIT_TEST_F(WriteToTopic_Demo_29, TFixture) +void TFixture::TestWriteToTopic29() { WriteMessagesInTx(1, 0); WriteMessagesInTx(1, 0); } -Y_UNIT_TEST_F(WriteToTopic_Demo_30, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_29_Table, TFixtureTable) +{ + TestWriteToTopic29(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_29_Query, TFixtureQuery) +{ + TestWriteToTopic29(); +} + +void TFixture::TestWriteToTopic30() { WriteMessagesInTx(1, 0); WriteMessagesInTx(0, 1); } -Y_UNIT_TEST_F(WriteToTopic_Demo_31, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_30_Table, TFixtureTable) +{ + TestWriteToTopic30(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_30_Query, TFixtureQuery) +{ + TestWriteToTopic30(); +} + +void TFixture::TestWriteToTopic31() { WriteMessagesInTx(1, 0); WriteMessagesInTx(1, 1); } -Y_UNIT_TEST_F(WriteToTopic_Demo_32, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_31_Table, TFixtureTable) +{ + TestWriteToTopic31(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_31_Query, TFixtureQuery) +{ + TestWriteToTopic31(); +} + +void TFixture::TestWriteToTopic32() { WriteMessagesInTx(0, 1); WriteMessagesInTx(1, 0); } -Y_UNIT_TEST_F(WriteToTopic_Demo_33, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_32_Table, TFixtureTable) +{ + TestWriteToTopic32(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_32_Query, TFixtureQuery) +{ + TestWriteToTopic32(); +} + +void TFixture::TestWriteToTopic33() { WriteMessagesInTx(0, 1); WriteMessagesInTx(0, 1); } -Y_UNIT_TEST_F(WriteToTopic_Demo_34, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_33_Table, TFixtureTable) +{ + TestWriteToTopic33(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_33_Query, TFixtureQuery) +{ + TestWriteToTopic33(); +} + +void TFixture::TestWriteToTopic34() { WriteMessagesInTx(0, 1); WriteMessagesInTx(1, 1); } -Y_UNIT_TEST_F(WriteToTopic_Demo_35, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_34_Table, TFixtureTable) +{ + TestWriteToTopic34(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_34_Query, TFixtureQuery) +{ + TestWriteToTopic34(); +} + +void TFixture::TestWriteToTopic35() { WriteMessagesInTx(1, 1); WriteMessagesInTx(1, 0); } -Y_UNIT_TEST_F(WriteToTopic_Demo_36, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_35_Table, TFixtureTable) +{ + TestWriteToTopic35(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_35_Query, TFixtureQuery) +{ + TestWriteToTopic35(); +} + +void TFixture::TestWriteToTopic36() { WriteMessagesInTx(1, 1); WriteMessagesInTx(0, 1); } -Y_UNIT_TEST_F(WriteToTopic_Demo_37, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_36_Table, TFixtureTable) +{ + TestWriteToTopic36(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_36_Query, TFixtureQuery) +{ + TestWriteToTopic36(); +} + +void TFixture::TestWriteToTopic37() { WriteMessagesInTx(1, 1); WriteMessagesInTx(1, 1); } +Y_UNIT_TEST_F(WriteToTopic_Demo_37_Table, TFixtureTable) +{ + TestWriteToTopic37(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_37_Query, TFixtureQuery) +{ + TestWriteToTopic37(); +} -Y_UNIT_TEST_F(WriteToTopic_Demo_38, TFixture) +void TFixture::TestWriteToTopic38() { WriteMessagesInTx(2, 202); WriteMessagesInTx(2, 200); @@ -2395,24 +3094,44 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_38, TFixture) WriteMessagesInTx(0, 1); } -Y_UNIT_TEST_F(WriteToTopic_Demo_39, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_38_Table, TFixtureTable) +{ + TestWriteToTopic38(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_38_Query, TFixtureQuery) +{ + TestWriteToTopic38(); +} + +void TFixture::TestWriteToTopic39() { CreateTopic("topic_A", TEST_CONSUMER); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); AddConsumer("topic_A", {"consumer"}); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); Read_Exactly_N_Messages_From_Topic("topic_A", "consumer", 2); } -Y_UNIT_TEST_F(ReadRuleGeneration, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_39_Table, TFixtureTable) +{ + TestWriteToTopic39(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_39_Query, TFixtureQuery) +{ + TestWriteToTopic39(); +} + +Y_UNIT_TEST_F(ReadRuleGeneration, TFixtureNoClient) { // There was a server NotifySchemeShard({.EnablePQConfigTransactionsAtSchemeShard = false}); @@ -2445,89 +3164,127 @@ Y_UNIT_TEST_F(ReadRuleGeneration, TFixture) Read_Exactly_N_Messages_From_Topic(TString{TEST_TOPIC}, "consumer-1", 1); } -Y_UNIT_TEST_F(WriteToTopic_Demo_40, TFixture) +void TFixture::TestWriteToTopic40() { // The recording stream will run into a quota. Before the commit, the client will receive confirmations // for some of the messages. The `CommitTx` call will wait for the rest. CreateTopic("topic_A", TEST_CONSUMER); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); for (size_t k = 0; k < 100; ++k) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), tx.get()); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 100); } -Y_UNIT_TEST_F(WriteToTopic_Demo_41, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_40_Table, TFixtureTable) +{ + TestWriteToTopic40(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_40_Query, TFixtureQuery) +{ + TestWriteToTopic40(); +} + +void TFixture::TestWriteToTopic41() { // If the recording session does not wait for confirmations, the commit will fail CreateTopic("topic_A", TEST_CONSUMER); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); for (size_t k = 0; k < 100; ++k) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), tx.get()); } CloseTopicWriteSession("topic_A", TEST_MESSAGE_GROUP_ID, true); // force close - CommitTx(tx, EStatus::SESSION_EXPIRED); + session->CommitTx(*tx, EStatus::SESSION_EXPIRED); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_41_Table, TFixtureTable) +{ + TestWriteToTopic41(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_41_Query, TFixtureQuery) +{ + TestWriteToTopic41(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_42, TFixture) +void TFixture::TestWriteToTopic42() { CreateTopic("topic_A", TEST_CONSUMER); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); for (size_t k = 0; k < 100; ++k) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), tx.get()); } CloseTopicWriteSession("topic_A", TEST_MESSAGE_GROUP_ID); // gracefully close - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 100); } -Y_UNIT_TEST_F(WriteToTopic_Demo_43, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_42_Table, TFixtureTable) +{ + TestWriteToTopic42(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_42_Query, TFixtureQuery) +{ + TestWriteToTopic42(); +} + +void TFixture::TestWriteToTopic43() { // The recording stream will run into a quota. Before the commit, the client will receive confirmations // for some of the messages. The `ExecuteDataQuery` call will wait for the rest. CreateTopic("topic_A", TEST_CONSUMER); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); for (size_t k = 0; k < 100; ++k) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), tx.get()); } - ExecuteDataQuery(tableSession, "SELECT 1", NTable::TTxControl::Tx(tx).CommitTx(true)); + session->Execute("SELECT 1", tx.get()); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 100); } -Y_UNIT_TEST_F(WriteToTopic_Demo_44, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_43_Table, TFixtureTable) { - CreateTopic("topic_A", TEST_CONSUMER); + TestWriteToTopic43(); +} - NTable::TSession tableSession = CreateTableSession(); +Y_UNIT_TEST_F(WriteToTopic_Demo_43_Query, TFixtureQuery) +{ + TestWriteToTopic43(); +} - auto result = ExecuteDataQuery(tableSession, "SELECT 1", NTable::TTxControl::BeginTx()); +void TFixture::TestWriteToTopic44() +{ + CreateTopic("topic_A", TEST_CONSUMER); + + auto session = CreateSession(); - NTable::TTransaction tx = *result.GetTransaction(); + auto [_, tx] = session->ExecuteInTx("SELECT 1", false); for (size_t k = 0; k < 100; ++k) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, TString(1'000'000, 'a'), tx.get()); } WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); @@ -2535,11 +3292,21 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_44, TFixture) auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(60)); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 0); - ExecuteDataQuery(tableSession, "SELECT 2", NTable::TTxControl::Tx(tx).CommitTx(true)); + session->Execute("SELECT 2", tx.get()); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 100); } +Y_UNIT_TEST_F(WriteToTopic_Demo_44_Table, TFixtureTable) +{ + TestWriteToTopic44(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_44_Query, TFixtureQuery) +{ + TestWriteToTopic44(); +} + void TFixture::CheckAvgWriteBytes(const TString& topicPath, ui32 partitionId, size_t minSize, size_t maxSize) @@ -2569,22 +3336,22 @@ void TFixture::SplitPartition(const TString& topicName, boundary); } -Y_UNIT_TEST_F(WriteToTopic_Demo_45, TFixture) +void TFixture::TestWriteToTopic45() { // Writing to a topic in a transaction affects the `AvgWriteBytes` indicator CreateTopic("topic_A", TEST_CONSUMER, 2); - auto session = CreateTableSession(); - auto tx = BeginTx(session); + auto session = CreateSession(); + auto tx = session->BeginTx(); TString message(1'000, 'x'); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, &tx, 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, &tx, 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, tx.get(), 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, tx.get(), 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, &tx, 1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, tx.get(), 1); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); size_t minSize = (message.size() + TEST_MESSAGE_GROUP_ID_1.size()) * 2; size_t maxSize = minSize + 200; @@ -2597,32 +3364,52 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_45, TFixture) CheckAvgWriteBytes("topic_A", 1, minSize, maxSize); } -Y_UNIT_TEST_F(WriteToTopic_Demo_46, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_45_Table, TFixtureTable) +{ + TestWriteToTopic45(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_45_Query, TFixtureQuery) +{ + TestWriteToTopic45(); +} + +void TFixture::TestWriteToTopic46() { // The `split` operation of the topic partition affects the writing in the transaction. // The transaction commit should fail with an error CreateTopic("topic_A", TEST_CONSUMER, 2, 10); - auto session = CreateTableSession(); - auto tx = BeginTx(session); + auto session = CreateSession(); + auto tx = session->BeginTx(); TString message(1'000, 'x'); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, &tx, 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, &tx, 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, tx.get(), 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, tx.get(), 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, &tx, 1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, tx.get(), 1); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_2); SplitPartition("topic_A", 1, "\xC0"); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, &tx, 1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, tx.get(), 1); + + session->CommitTx(*tx, EStatus::ABORTED); +} - CommitTx(tx, EStatus::ABORTED); +Y_UNIT_TEST_F(WriteToTopic_Demo_46_Table, TFixtureTable) +{ + TestWriteToTopic46(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_46_Query, TFixtureQuery) +{ + TestWriteToTopic46(); } -Y_UNIT_TEST_F(WriteToTopic_Demo_47, TFixture) +void TFixture::TestWriteToTopic47() { // The `split` operation of the topic partition does not affect the reading in the transaction. CreateTopic("topic_A", TEST_CONSUMER, 2, 10); @@ -2640,42 +3427,52 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_47, TFixture) SplitPartition("topic_A", 1, "\xC0"); - auto session = CreateTableSession(); - auto tx = BeginTx(session); + auto session = CreateSession(); + auto tx = session->BeginTx(); - auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), &tx, 0); + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), tx.get(), 0); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); CloseTopicReadSession("topic_A", TEST_CONSUMER); - messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), &tx, 1); + messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), tx.get(), 1); UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); } -Y_UNIT_TEST_F(WriteToTopic_Demo_48, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_47_Table, TFixtureTable) +{ + TestWriteToTopic47(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_47_Query, TFixtureQuery) +{ + TestWriteToTopic47(); +} + +void TFixture::TestWriteToTopic48() { // the commit of a transaction affects the split of the partition CreateTopic("topic_A", TEST_CONSUMER, 2, 10); AlterAutoPartitioning("topic_A", 2, 10, EAutoPartitioningStrategy::ScaleUp, TDuration::Seconds(2), 1, 2); - auto session = CreateTableSession(); - auto tx = BeginTx(session); + auto session = CreateSession(); + auto tx = session->BeginTx(); TString message(1_MB, 'x'); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, &tx, 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, &tx, 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_3, message, &tx, 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_3, message, &tx, 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, tx.get(), 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message, tx.get(), 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_3, message, tx.get(), 0); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_3, message, tx.get(), 0); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, &tx, 1); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, &tx, 1); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_4, message, &tx, 1); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_4, message, &tx, 1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, tx.get(), 1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, tx.get(), 1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_4, message, tx.get(), 1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_4, message, tx.get(), 1); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); Sleep(TDuration::Seconds(5)); @@ -2684,7 +3481,17 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_48, TFixture) UNIT_ASSERT_GT(topicDescription.GetTotalPartitionsCount(), 2); } -Y_UNIT_TEST_F(WriteToTopic_Demo_50, TFixture) +Y_UNIT_TEST_F(WriteToTopic_Demo_48_Table, TFixtureTable) +{ + TestWriteToTopic48(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_48_Query, TFixtureQuery) +{ + TestWriteToTopic48(); +} + +void TFixture::TestWriteToTopic50() { // We write to the topic in the transaction. When a transaction is committed, the keys in the blob // cache are renamed. @@ -2696,19 +3503,19 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_50, TFixture) WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, message); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_1); - auto session = CreateTableSession(); + auto session = CreateSession(); // tx #1 // After the transaction commit, there will be no large blobs in the batches. The number of renames // will not change in the cache. - auto tx = BeginTx(session); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_3, message, &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_3, message, tx.get()); UNIT_ASSERT_VALUES_EQUAL(GetPQCacheRenameKeysCount(), 0); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); Sleep(TDuration::Seconds(5)); @@ -2716,23 +3523,33 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_50, TFixture) // tx #2 // After the commit, the party will rename one big blob - tx = BeginTx(session); + tx = session->BeginTx(); for (unsigned i = 0; i < 80; ++i) { - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, message, tx.get()); } - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_3, message, &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID_3, message, tx.get()); UNIT_ASSERT_VALUES_EQUAL(GetPQCacheRenameKeysCount(), 0); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); Sleep(TDuration::Seconds(5)); UNIT_ASSERT_VALUES_EQUAL(GetPQCacheRenameKeysCount(), 1); } +Y_UNIT_TEST_F(WriteToTopic_Demo_50_Table, TFixtureTable) +{ + TestWriteToTopic50(); +} + +Y_UNIT_TEST_F(WriteToTopic_Demo_50_Query, TFixtureQuery) +{ + TestWriteToTopic50(); +} + class TFixtureSinks : public TFixture { protected: void CreateRowTable(const TString& path); @@ -2742,6 +3559,31 @@ class TFixtureSinks : public TFixture { bool GetEnableOlapSink() const override; bool GetEnableHtapTx() const override; bool GetAllowOlapDataQuery() const override; + + void TestSinksOltpWriteToTopic5(); + + void TestSinksOltpWriteToTopicAndTable2(); + void TestSinksOltpWriteToTopicAndTable3(); + void TestSinksOltpWriteToTopicAndTable4(); + void TestSinksOltpWriteToTopicAndTable5(); + + void TestSinksOlapWriteToTopicAndTable1(); + void TestSinksOlapWriteToTopicAndTable2(); + void TestSinksOlapWriteToTopicAndTable3(); +}; + +class TFixtureSinksTable : public TFixtureSinks { +protected: + EClientType GetClientType() const override { + return EClientType::Table; + } +}; + +class TFixtureSinksQuery : public TFixtureSinks { +protected: + EClientType GetClientType() const override { + return EClientType::Query; + } }; void TFixtureSinks::CreateRowTable(const TString& path) @@ -2755,7 +3597,10 @@ void TFixtureSinks::CreateColumnTable(const TString& tablePath) TString path = (tablePath[0] != '/') ? ("/Root/" + tablePath) : tablePath; - NTable::TSession session = CreateTableSession(); + auto createSessionResult = GetTableClient().CreateSession().ExtractValueSync(); + UNIT_ASSERT_C(createSessionResult.IsSuccess(), createSessionResult.GetIssues().ToString()); + auto session = createSessionResult.GetSession(); + auto desc = NTable::TTableBuilder() .SetStoreType(NTable::EStoreType::Column) .AddNonNullableColumn("key", EPrimitiveType::Utf8) @@ -2786,88 +3631,143 @@ bool TFixtureSinks::GetAllowOlapDataQuery() const return true; } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_1, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_1_Table, TFixtureSinksTable) { TestWriteToTopic7(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_2, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_1_Query, TFixtureSinksQuery) +{ + TestWriteToTopic7(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_2_Table, TFixtureSinksTable) +{ + TestWriteToTopic10(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_2_Query, TFixtureSinksQuery) { TestWriteToTopic10(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_3, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_3_Table, TFixtureSinksTable) +{ + TestWriteToTopic26(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_3_Query, TFixtureSinksQuery) { TestWriteToTopic26(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_4, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_4_Table, TFixtureSinksTable) +{ + TestWriteToTopic9(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_4_Query, TFixtureSinksQuery) { TestWriteToTopic9(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_5, TFixtureSinks) +void TFixtureSinks::TestSinksOltpWriteToTopic5() { CreateTopic("topic_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 0); - RollbackTx(tx, EStatus::SUCCESS); + session->RollbackTx(*tx, EStatus::SUCCESS); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 0); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_1, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_5_Table, TFixtureSinksTable) +{ + TestSinksOltpWriteToTopic5(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopic_5_Query, TFixtureSinksQuery) +{ + TestSinksOltpWriteToTopic5(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_1_Table, TFixtureSinksTable) +{ + TestWriteToTopic1(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_1_Query, TFixtureSinksQuery) { TestWriteToTopic1(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_2, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_2_Table, TFixtureSinksTable) { TestWriteToTopic27(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_3, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_2_Query, TFixtureSinksQuery) +{ + TestWriteToTopic27(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_3_Table, TFixtureSinksTable) { TestWriteToTopic11(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_4, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_3_Query, TFixtureSinksQuery) +{ + TestWriteToTopic11(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_4_Table, TFixtureSinksTable) +{ + TestWriteToTopic4(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopics_4_Query, TFixtureSinksQuery) { TestWriteToTopic4(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_1, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_1_Table, TFixtureSinksTable) +{ + TestWriteToTopic24(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_1_Query, TFixtureSinksQuery) { TestWriteToTopic24(); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_2, TFixtureSinks) +void TFixtureSinks::TestSinksOltpWriteToTopicAndTable2() { CreateTopic("topic_A"); CreateTopic("topic_B"); CreateRowTable("/Root/table_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, &tx); + WriteToTable("table_A", records, *session, tx.get()); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #1", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", &tx); - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #3", &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #3", tx.get()); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); { auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 1); @@ -2886,7 +3786,17 @@ Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_2, TFixtureSinks) CheckTabletKeys("topic_B"); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_3, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_2_Table, TFixtureSinksTable) +{ + TestSinksOltpWriteToTopicAndTable2(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_2_Query, TFixtureSinksQuery) +{ + TestSinksOltpWriteToTopicAndTable2(); +} + +void TFixtureSinks::TestSinksOltpWriteToTopicAndTable3() { CreateTopic("topic_A"); CreateTopic("topic_B"); @@ -2894,21 +3804,21 @@ Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_3, TFixtureSinks) CreateRowTable("/Root/table_A"); CreateRowTable("/Root/table_B"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, &tx); - WriteToTable("table_B", records, &tx); + WriteToTable("table_A", records, *session, tx.get()); + WriteToTable("table_B", records, *session, tx.get()); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); const size_t topicMsgCnt = 10; for (size_t i = 1; i <= topicMsgCnt; ++i) { - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #" + std::to_string(i), &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #" + std::to_string(i), tx.get()); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); { auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 1); @@ -2928,25 +3838,35 @@ Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_3, TFixtureSinks) CheckTabletKeys("topic_B"); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_4, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_3_Table, TFixtureSinksTable) +{ + TestSinksOltpWriteToTopicAndTable3(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_3_Query, TFixtureSinksQuery) +{ + TestSinksOltpWriteToTopicAndTable3(); +} + +void TFixtureSinks::TestSinksOltpWriteToTopicAndTable4() { CreateTopic("topic_A"); CreateRowTable("/Root/table_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx1 = BeginTx(tableSession); - NTable::TTransaction tx2 = BeginTx(tableSession); + auto session = CreateSession(); + auto tx1 = session->BeginTx(); + auto tx2 = session->BeginTx(); - ExecuteDataQuery(tableSession, R"(SELECT COUNT(*) FROM `table_A`)", NTable::TTxControl::Tx(tx1)); + session->Execute(R"(SELECT COUNT(*) FROM `table_A`)", tx1.get(), false); auto records = MakeTableRecords(); - WriteToTable("table_A", records, &tx2); + WriteToTable("table_A", records, *session, tx2.get()); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx1); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx1.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); - CommitTx(tx2, EStatus::SUCCESS); - CommitTx(tx1, EStatus::ABORTED); + session->CommitTx(*tx2, EStatus::SUCCESS); + session->CommitTx(*tx1, EStatus::ABORTED); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 0); @@ -2955,21 +3875,31 @@ Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_4, TFixtureSinks) CheckTabletKeys("topic_A"); } -Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_5, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_4_Table, TFixtureSinksTable) +{ + TestSinksOltpWriteToTopicAndTable4(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_4_Query, TFixtureSinksQuery) +{ + TestSinksOltpWriteToTopicAndTable4(); +} + +void TFixtureSinks::TestSinksOltpWriteToTopicAndTable5() { CreateTopic("topic_A"); CreateRowTable("/Root/table_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, &tx); + WriteToTable("table_A", records, *session, tx.get()); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); - RollbackTx(tx, EStatus::SUCCESS); + session->RollbackTx(*tx, EStatus::SUCCESS); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 0); @@ -2978,20 +3908,30 @@ Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_5, TFixtureSinks) CheckTabletKeys("topic_A"); } -Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_1, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_5_Table, TFixtureSinksTable) +{ + TestSinksOltpWriteToTopicAndTable5(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_5_Query, TFixtureSinksQuery) +{ + TestSinksOltpWriteToTopicAndTable5(); +} + +void TFixtureSinks::TestSinksOlapWriteToTopicAndTable1() { return; // https://github.com/ydb-platform/ydb/issues/17271 CreateTopic("topic_A"); CreateColumnTable("/Root/table_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, &tx); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx); + WriteToTable("table_A", records, *session, tx.get()); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 1); UNIT_ASSERT_VALUES_EQUAL(messages.front(), MakeJsonDoc(records)); @@ -3001,7 +3941,17 @@ Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_1, TFixtureSinks) CheckTabletKeys("topic_A"); } -Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_2, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_1_Table, TFixtureSinksTable) +{ + TestSinksOlapWriteToTopicAndTable1(); +} + +Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_1_Query, TFixtureSinksQuery) +{ + TestSinksOlapWriteToTopicAndTable1(); +} + +void TFixtureSinks::TestSinksOlapWriteToTopicAndTable2() { return; // https://github.com/ydb-platform/ydb/issues/17271 CreateTopic("topic_A"); @@ -3010,22 +3960,22 @@ Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_2, TFixtureSinks) CreateRowTable("/Root/table_A"); CreateColumnTable("/Root/table_B"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, &tx); - WriteToTable("table_B", records, &tx); + WriteToTable("table_A", records, *session, tx.get()); + WriteToTable("table_B", records, *session, tx.get()); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); const size_t topicMsgCnt = 10; for (size_t i = 1; i <= topicMsgCnt; ++i) { - WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #" + std::to_string(i), &tx); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #" + std::to_string(i), tx.get()); } - CommitTx(tx, EStatus::SUCCESS); + session->CommitTx(*tx, EStatus::SUCCESS); { auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 1); @@ -3045,21 +3995,31 @@ Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_2, TFixtureSinks) CheckTabletKeys("topic_B"); } -Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_3, TFixtureSinks) +Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_2_Table, TFixtureSinksTable) +{ + TestSinksOlapWriteToTopicAndTable2(); +} + +Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_2_Query, TFixtureSinksQuery) +{ + TestSinksOlapWriteToTopicAndTable2(); +} + +void TFixtureSinks::TestSinksOlapWriteToTopicAndTable3() { CreateTopic("topic_A"); CreateColumnTable("/Root/table_A"); - NTable::TSession tableSession = CreateTableSession(); - NTable::TTransaction tx = BeginTx(tableSession); + auto session = CreateSession(); + auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, &tx); + WriteToTable("table_A", records, *session, tx.get()); - WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), &tx); + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); - RollbackTx(tx, EStatus::SUCCESS); + session->RollbackTx(*tx, EStatus::SUCCESS); Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 0); @@ -3068,7 +4028,17 @@ Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_3, TFixtureSinks) CheckTabletKeys("topic_A"); } -Y_UNIT_TEST_F(Write_Random_Sized_Messages_In_Wide_Transactions, TFixture) +Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_3_Table, TFixtureSinksTable) +{ + TestSinksOlapWriteToTopicAndTable3(); +} + +Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_3_Query, TFixtureSinksQuery) +{ + TestSinksOlapWriteToTopicAndTable3(); +} + +void TFixture::TestWriteRandomSizedMessagesInWideTransactions() { // The test verifies the simultaneous execution of several transactions. There is a topic // with PARTITIONS_COUNT partitions. In each transaction, the test writes to all the partitions. @@ -3083,15 +4053,15 @@ Y_UNIT_TEST_F(Write_Random_Sized_Messages_In_Wide_Transactions, TFixture) SetPartitionWriteSpeed("topic_A", 50'000'000); - std::vector sessions; - std::vector transactions; + std::vector> sessions; + std::vector> transactions; // We open TXS_COUNT transactions and write messages to the topic. for (size_t i = 0; i < TXS_COUNT; ++i) { - sessions.push_back(CreateTableSession()); + sessions.push_back(CreateSession()); auto& session = sessions.back(); - transactions.push_back(BeginTx(session)); + transactions.push_back(session->BeginTx()); auto& tx = transactions.back(); for (size_t j = 0; j < PARTITIONS_COUNT; ++j) { @@ -3102,17 +4072,17 @@ Y_UNIT_TEST_F(Write_Random_Sized_Messages_In_Wide_Transactions, TFixture) sourceId += ToString(j); size_t count = RandomNumber(20) + 3; - WriteToTopic("topic_A", sourceId, TString(512 * 1000 * count, 'x'), &tx, j); + WriteToTopic("topic_A", sourceId, TString(512 * 1000 * count, 'x'), tx.get(), j); WaitForAcks("topic_A", sourceId); } } // We are doing an asynchronous commit of transactions. They will be executed simultaneously. - std::vector futures; + std::vector futures; for (size_t i = 0; i < TXS_COUNT; ++i) { - futures.push_back(transactions[i].Commit()); + futures.push_back(sessions[i]->AsyncCommitTx(*transactions[i])); } // All transactions must be completed successfully. @@ -3123,7 +4093,17 @@ Y_UNIT_TEST_F(Write_Random_Sized_Messages_In_Wide_Transactions, TFixture) } } -Y_UNIT_TEST_F(Write_Only_Big_Messages_In_Wide_Transactions, TFixture) +Y_UNIT_TEST_F(Write_Random_Sized_Messages_In_Wide_Transactions_Table, TFixtureTable) +{ + TestWriteRandomSizedMessagesInWideTransactions(); +} + +Y_UNIT_TEST_F(Write_Random_Sized_Messages_In_Wide_Transactions_Query, TFixtureQuery) +{ + TestWriteRandomSizedMessagesInWideTransactions(); +} + +void TFixture::TestWriteOnlyBigMessagesInWideTransactions() { // The test verifies the simultaneous execution of several transactions. There is a topic `topic_A` and // it contains a `PARTITIONS_COUNT' of partitions. In each transaction, the test writes to all partitions. @@ -3137,15 +4117,15 @@ Y_UNIT_TEST_F(Write_Only_Big_Messages_In_Wide_Transactions, TFixture) SetPartitionWriteSpeed("topic_A", 50'000'000); - std::vector sessions; - std::vector transactions; + std::vector> sessions; + std::vector> transactions; // We open TXS_COUNT transactions and write messages to the topic. for (size_t i = 0; i < TXS_COUNT; ++i) { - sessions.push_back(CreateTableSession()); + sessions.push_back(CreateSession()); auto& session = sessions.back(); - transactions.push_back(BeginTx(session)); + transactions.push_back(session->BeginTx()); auto& tx = transactions.back(); for (size_t j = 0; j < PARTITIONS_COUNT; ++j) { @@ -3155,17 +4135,17 @@ Y_UNIT_TEST_F(Write_Only_Big_Messages_In_Wide_Transactions, TFixture) sourceId += "_"; sourceId += ToString(j); - WriteToTopic("topic_A", sourceId, TString(6'500'000, 'x'), &tx, j); + WriteToTopic("topic_A", sourceId, TString(6'500'000, 'x'), tx.get(), j); WaitForAcks("topic_A", sourceId); } } // We are doing an asynchronous commit of transactions. They will be executed simultaneously. - std::vector futures; + std::vector futures; for (size_t i = 0; i < TXS_COUNT; ++i) { - futures.push_back(transactions[i].Commit()); + futures.push_back(sessions[i]->AsyncCommitTx(*transactions[i])); } // All transactions must be completed successfully. @@ -3176,7 +4156,17 @@ Y_UNIT_TEST_F(Write_Only_Big_Messages_In_Wide_Transactions, TFixture) } } -Y_UNIT_TEST_F(Transactions_Conflict_On_SeqNo, TFixture) +Y_UNIT_TEST_F(Write_Only_Big_Messages_In_Wide_Transactions_Table, TFixtureTable) +{ + TestWriteOnlyBigMessagesInWideTransactions(); +} + +Y_UNIT_TEST_F(Write_Only_Big_Messages_In_Wide_Transactions_Query, TFixtureQuery) +{ + TestWriteOnlyBigMessagesInWideTransactions(); +} + +void TFixture::TestTransactionsConflictOnSeqNo() { const ui32 PARTITIONS_COUNT = 20; const size_t TXS_COUNT = 100; @@ -3185,7 +4175,7 @@ Y_UNIT_TEST_F(Transactions_Conflict_On_SeqNo, TFixture) SetPartitionWriteSpeed("topic_A", 50'000'000); - auto tableSession = CreateTableSession(); + auto session = CreateSession(); std::vector> topicWriteSessions; for (ui32 i = 0; i < PARTITIONS_COUNT; ++i) { @@ -3206,14 +4196,14 @@ Y_UNIT_TEST_F(Transactions_Conflict_On_SeqNo, TFixture) topicWriteSessions.push_back(std::move(session)); } - std::vector sessions; - std::vector transactions; + std::vector> sessions; + std::vector> transactions; for (size_t i = 0; i < TXS_COUNT; ++i) { - sessions.push_back(CreateTableSession()); + sessions.push_back(CreateSession()); auto& session = sessions.back(); - transactions.push_back(BeginTx(session)); + transactions.push_back(session->BeginTx()); auto& tx = transactions.back(); for (size_t j = 0; j < PARTITIONS_COUNT; ++j) { @@ -3224,17 +4214,17 @@ Y_UNIT_TEST_F(Transactions_Conflict_On_SeqNo, TFixture) for (size_t k = 0, count = RandomNumber(20) + 1; k < count; ++k) { const std::string data(RandomNumber(1'000) + 100, 'x'); NTopic::TWriteMessage params(data); - params.Tx(tx); + params.Tx(*tx); topicWriteSessions[j]->Write(std::move(params)); } } } - std::vector futures; + std::vector futures; for (size_t i = 0; i < TXS_COUNT; ++i) { - futures.push_back(transactions[i].Commit()); + futures.push_back(sessions[i]->AsyncCommitTx(*transactions[i])); } // Some transactions should end with the error `ABORTED` @@ -3258,7 +4248,17 @@ Y_UNIT_TEST_F(Transactions_Conflict_On_SeqNo, TFixture) UNIT_ASSERT_VALUES_UNEQUAL(successCount, TXS_COUNT); } -Y_UNIT_TEST_F(The_Transaction_Starts_On_One_Version_And_Ends_On_The_Other, TFixture) +Y_UNIT_TEST_F(Transactions_Conflict_On_SeqNo_Table, TFixtureTable) +{ + TestTransactionsConflictOnSeqNo(); +} + +Y_UNIT_TEST_F(Transactions_Conflict_On_SeqNo_Query, TFixtureQuery) +{ + TestTransactionsConflictOnSeqNo(); +} + +Y_UNIT_TEST_F(The_Transaction_Starts_On_One_Version_And_Ends_On_The_Other, TFixtureNoClient) { // In the test, we check the compatibility between versions `24-4-2` and `24-4-*/25-1-*`. To do this, the data // obtained on the `24-4-2` version is loaded into the PQ tablets. diff --git a/tests/integration/server_restart/main.cpp b/tests/integration/server_restart/main.cpp index fafc564c93..08572b9f6a 100644 --- a/tests/integration/server_restart/main.cpp +++ b/tests/integration/server_restart/main.cpp @@ -81,7 +81,7 @@ class TQueryProxy final : public Ydb::Query::V1::QueryService::Service { template grpc::Status RunStream(TGrpcStreamCall call, grpc::ServerContext *context, - const TRequest* request, grpc::ServerWriter* writer) { + const TRequest* request, grpc::ServerWriter* writer) { auto clientContext = grpc::ClientContext::FromServerContext(*context); auto reader = (Stub_.*call)(clientContext.get(), *request); From fd7419c78b8e3b084ecaa4e5902092bab89d27db Mon Sep 17 00:00:00 2001 From: Evgeniy Ivanov Date: Fri, 23 May 2025 14:23:58 +0000 Subject: [PATCH 02/16] Print additional latencies for GRPC ping (#17683) --- src/api/protos/ydb_debug.proto | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/protos/ydb_debug.proto b/src/api/protos/ydb_debug.proto index aefd20f6a7..1f77495ee2 100644 --- a/src/api/protos/ydb_debug.proto +++ b/src/api/protos/ydb_debug.proto @@ -16,6 +16,8 @@ message PlainGrpcRequest { message PlainGrpcResponse { StatusIds.StatusCode status = 1; repeated Ydb.Issue.IssueMessage issues = 2; + + uint64 CallBackTs = 3; } // Go until GrpcProxy From faeaeb23c35d08b3a7a2e9a971194acc68224d9a Mon Sep 17 00:00:00 2001 From: Vasily Gerasimov Date: Fri, 23 May 2025 14:24:56 +0000 Subject: [PATCH 03/16] Export/import encryption parameters in YDB SDK (#17798) --- include/ydb-cpp-sdk/client/export/export.h | 17 ++++++++++++++ include/ydb-cpp-sdk/client/import/import.h | 13 +++++++++++ src/client/export/export.cpp | 21 +++++++++++++++++ src/client/import/import.cpp | 26 ++++++++++++++++++++-- 4 files changed, 75 insertions(+), 2 deletions(-) diff --git a/include/ydb-cpp-sdk/client/export/export.h b/include/ydb-cpp-sdk/client/export/export.h index 2f87c880da..bbda145988 100644 --- a/include/ydb-cpp-sdk/client/export/export.h +++ b/include/ydb-cpp-sdk/client/export/export.h @@ -79,6 +79,12 @@ struct TExportToS3Settings : public TOperationRequestSettings::max(), }; + struct TEncryptionAlgorithm { + static const std::string AES_128_GCM; + static const std::string AES_256_GCM; + static const std::string CHACHA_20_POLY_1305; + }; + struct TItem { std::string Src; std::string Dst; @@ -89,6 +95,17 @@ struct TExportToS3Settings : public TOperationRequestSettings TExportClient::ExportToS3(const TExportToS3Settings request.mutable_settings()->set_compression(TStringType{settings.Compression_.value()}); } + if (settings.SourcePath_) { + request.mutable_settings()->set_source_path(settings.SourcePath_.value()); + } + + if (settings.DestinationPrefix_) { + request.mutable_settings()->set_destination_prefix(settings.DestinationPrefix_.value()); + } + request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_); + if (settings.EncryptionAlgorithm_.empty() != settings.SymmetricKey_.empty()) { + throw TContractViolation("Encryption algorithm and symmetric key must be set together"); + } + + if (!settings.EncryptionAlgorithm_.empty() && !settings.SymmetricKey_.empty()) { + request.mutable_settings()->mutable_encryption_settings()->set_encryption_algorithm(settings.EncryptionAlgorithm_); + request.mutable_settings()->mutable_encryption_settings()->mutable_symmetric_key()->set_key(settings.SymmetricKey_); + } + return Impl_->ExportToS3(std::move(request), settings); } diff --git a/src/client/import/import.cpp b/src/client/import/import.cpp index fe8062a3c7..506841959d 100644 --- a/src/client/import/import.cpp +++ b/src/client/import/import.cpp @@ -141,9 +141,19 @@ TFuture TImportClient::ImportFromS3(const TImportFromS3Se request.mutable_settings()->set_secret_key(TStringType{settings.SecretKey_}); for (const auto& item : settings.Item_) { + if (!item.Src.empty() && !item.SrcPath.empty()) { + throw TContractViolation( + TStringBuilder() << "Invalid item: both source prefix and source path are set: \"" << item.Src << "\" and \"" << item.SrcPath << "\""); + } + auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); - protoItem.set_source_prefix(TStringType{item.Src}); - protoItem.set_destination_path(TStringType{item.Dst}); + if (!item.Src.empty()) { + protoItem.set_source_prefix(item.Src); + } + if (!item.SrcPath.empty()) { + protoItem.set_source_path(item.SrcPath); + } + protoItem.set_destination_path(item.Dst); } if (settings.Description_) { @@ -158,6 +168,18 @@ TFuture TImportClient::ImportFromS3(const TImportFromS3Se request.mutable_settings()->set_no_acl(settings.NoACL_.value()); } + if (settings.SourcePrefix_) { + request.mutable_settings()->set_source_prefix(settings.SourcePrefix_.value()); + } + + if (settings.DestinationPath_) { + request.mutable_settings()->set_destination_path(settings.DestinationPath_.value()); + } + + if (settings.SymmetricKey_) { + request.mutable_settings()->mutable_encryption_settings()->mutable_symmetric_key()->set_key(*settings.SymmetricKey_); + } + request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_); return Impl_->ImportFromS3(std::move(request), settings); From 69efbfcf2157493f6f5da7287215549fde8b5218 Mon Sep 17 00:00:00 2001 From: Nikita Vasilev Date: Fri, 23 May 2025 14:26:34 +0000 Subject: [PATCH 04/16] Fix hasRead for sink (#17844) --- src/client/topic/ut/topic_to_table_ut.cpp | 184 ++++++++++++++++++++-- 1 file changed, 172 insertions(+), 12 deletions(-) diff --git a/src/client/topic/ut/topic_to_table_ut.cpp b/src/client/topic/ut/topic_to_table_ut.cpp index f2ce3569aa..89deb6a7de 100644 --- a/src/client/topic/ut/topic_to_table_ut.cpp +++ b/src/client/topic/ut/topic_to_table_ut.cpp @@ -197,7 +197,15 @@ class TFixture : public NUnitTest::TBaseFixture { TString MakeJsonDoc(const TVector& records); void CreateTable(const TString& path); - void WriteToTable(const TString& tablePath, + void UpsertToTable(const TString& tablePath, + const TVector& records, + ISession& session, + TTransactionBase* tx); + void InsertToTable(const TString& tablePath, + const TVector& records, + ISession& session, + TTransactionBase* tx); + void DeleteFromTable(const TString& tablePath, const TVector& records, ISession& session, TTransactionBase* tx); @@ -1795,7 +1803,7 @@ void TFixture::TestWriteToTopic24() auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, *session, tx.get()); + UpsertToTable("table_A", records, *session, tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); session->CommitTx(*tx, EStatus::SUCCESS); @@ -2790,7 +2798,7 @@ auto TFixture::MakeJsonDoc(const TVector& records) -> TString return s; } -void TFixture::WriteToTable(const TString& tablePath, +void TFixture::UpsertToTable(const TString& tablePath, const TVector& records, ISession& session, TTransactionBase* tx) @@ -2810,6 +2818,46 @@ void TFixture::WriteToTable(const TString& tablePath, } } +void TFixture::InsertToTable(const TString& tablePath, + const TVector& records, + ISession& session, + TTransactionBase* tx) +{ + TString query = Sprintf("DECLARE $key AS Utf8;" + "DECLARE $value AS Utf8;" + "INSERT INTO `%s` (key, value) VALUES ($key, $value);", + tablePath.data()); + + for (const auto& r : records) { + auto params = TParamsBuilder() + .AddParam("$key").Utf8(r.Key).Build() + .AddParam("$value").Utf8(r.Value).Build() + .Build(); + + session.Execute(query, tx, false, params); + } +} + +void TFixture::DeleteFromTable(const TString& tablePath, + const TVector& records, + ISession& session, + TTransactionBase* tx) +{ + TString query = Sprintf("DECLARE $key AS Utf8;" + "DECLARE $value AS Utf8;" + "DELETE FROM `%s` ON (key, value) VALUES ($key, $value);", + tablePath.data()); + + for (const auto& r : records) { + auto params = TParamsBuilder() + .AddParam("$key").Utf8(r.Key).Build() + .AddParam("$value").Utf8(r.Value).Build() + .Build(); + + session.Execute(query, tx, false, params); + } +} + size_t TFixture::GetTableRecordsCount(const TString& tablePath) { TString query = Sprintf(R"(SELECT COUNT(*) FROM `%s`)", @@ -3566,10 +3614,12 @@ class TFixtureSinks : public TFixture { void TestSinksOltpWriteToTopicAndTable3(); void TestSinksOltpWriteToTopicAndTable4(); void TestSinksOltpWriteToTopicAndTable5(); + void TestSinksOltpWriteToTopicAndTable6(); void TestSinksOlapWriteToTopicAndTable1(); void TestSinksOlapWriteToTopicAndTable2(); void TestSinksOlapWriteToTopicAndTable3(); + void TestSinksOlapWriteToTopicAndTable4(); }; class TFixtureSinksTable : public TFixtureSinks { @@ -3759,7 +3809,7 @@ void TFixtureSinks::TestSinksOltpWriteToTopicAndTable2() auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, *session, tx.get()); + UpsertToTable("table_A", records, *session, tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); @@ -3786,6 +3836,7 @@ void TFixtureSinks::TestSinksOltpWriteToTopicAndTable2() CheckTabletKeys("topic_B"); } + Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_2_Table, TFixtureSinksTable) { TestSinksOltpWriteToTopicAndTable2(); @@ -3808,8 +3859,8 @@ void TFixtureSinks::TestSinksOltpWriteToTopicAndTable3() auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, *session, tx.get()); - WriteToTable("table_B", records, *session, tx.get()); + UpsertToTable("table_A", records, *session, tx.get()); + UpsertToTable("table_B", records, *session, tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); @@ -3860,7 +3911,7 @@ void TFixtureSinks::TestSinksOltpWriteToTopicAndTable4() session->Execute(R"(SELECT COUNT(*) FROM `table_A`)", tx1.get(), false); auto records = MakeTableRecords(); - WriteToTable("table_A", records, *session, tx2.get()); + UpsertToTable("table_A", records, *session, tx2.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx1.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); @@ -3894,7 +3945,7 @@ void TFixtureSinks::TestSinksOltpWriteToTopicAndTable5() auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, *session, tx.get()); + UpsertToTable("table_A", records, *session, tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); @@ -3918,6 +3969,56 @@ Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_5_Query, TFixtureSinksQuery) TestSinksOltpWriteToTopicAndTable5(); } +void TFixtureSinks::TestSinksOltpWriteToTopicAndTable6() +{ + CreateTopic("topic_A"); + CreateTopic("topic_B"); + CreateRowTable("/Root/table_A"); + + auto session = CreateSession(); + auto tx = session->BeginTx(); + + auto records = MakeTableRecords(); + InsertToTable("table_A", records, *session, tx.get()); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); + + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #1", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #2", tx.get()); + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #3", tx.get()); + + DeleteFromTable("table_A", records, *session, tx.get()); + + session->CommitTx(*tx, EStatus::SUCCESS); + + { + auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 1); + UNIT_ASSERT_VALUES_EQUAL(messages.front(), MakeJsonDoc(records)); + } + + { + auto messages = Read_Exactly_N_Messages_From_Topic("topic_B", TEST_CONSUMER, 3); + UNIT_ASSERT_VALUES_EQUAL(messages.front(), "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages.back(), "message #3"); + } + + UNIT_ASSERT_VALUES_EQUAL(GetTableRecordsCount("table_A"), 0); + + CheckTabletKeys("topic_A"); + CheckTabletKeys("topic_B"); +} + + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_6_Table, TFixtureSinksTable) +{ + TestSinksOltpWriteToTopicAndTable6(); +} + +Y_UNIT_TEST_F(Sinks_Oltp_WriteToTopicAndTable_6_Query, TFixtureSinksQuery) +{ + TestSinksOltpWriteToTopicAndTable6(); +} + void TFixtureSinks::TestSinksOlapWriteToTopicAndTable1() { return; // https://github.com/ydb-platform/ydb/issues/17271 @@ -3928,7 +4029,7 @@ void TFixtureSinks::TestSinksOlapWriteToTopicAndTable1() auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, *session, tx.get()); + UpsertToTable("table_A", records, *session, tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); session->CommitTx(*tx, EStatus::SUCCESS); @@ -3965,8 +4066,8 @@ void TFixtureSinks::TestSinksOlapWriteToTopicAndTable2() auto records = MakeTableRecords(); - WriteToTable("table_A", records, *session, tx.get()); - WriteToTable("table_B", records, *session, tx.get()); + UpsertToTable("table_A", records, *session, tx.get()); + UpsertToTable("table_B", records, *session, tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); @@ -4014,7 +4115,7 @@ void TFixtureSinks::TestSinksOlapWriteToTopicAndTable3() auto tx = session->BeginTx(); auto records = MakeTableRecords(); - WriteToTable("table_A", records, *session, tx.get()); + UpsertToTable("table_A", records, *session, tx.get()); WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID); @@ -4038,6 +4139,65 @@ Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_3_Query, TFixtureSinksQuery) TestSinksOlapWriteToTopicAndTable3(); } +void TFixtureSinks::TestSinksOlapWriteToTopicAndTable4() +{ + return; // https://github.com/ydb-platform/ydb/issues/17271 + CreateTopic("topic_A"); + CreateTopic("topic_B"); + + CreateRowTable("/Root/table_A"); + CreateColumnTable("/Root/table_B"); + CreateColumnTable("/Root/table_C"); + + auto session = CreateSession(); + auto tx = session->BeginTx(); + + auto records = MakeTableRecords(); + + InsertToTable("table_A", records, *session, tx.get()); + InsertToTable("table_B", records, *session, tx.get()); + UpsertToTable("table_C", records, *session, tx.get()); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID, MakeJsonDoc(records), tx.get()); + + const size_t topicMsgCnt = 10; + for (size_t i = 1; i <= topicMsgCnt; ++i) { + WriteToTopic("topic_B", TEST_MESSAGE_GROUP_ID, "message #" + std::to_string(i), tx.get()); + } + + DeleteFromTable("table_B", records, *session, tx.get()); + + session->CommitTx(*tx, EStatus::SUCCESS); + + { + auto messages = Read_Exactly_N_Messages_From_Topic("topic_A", TEST_CONSUMER, 1); + UNIT_ASSERT_VALUES_EQUAL(messages.front(), MakeJsonDoc(records)); + } + + { + auto messages = Read_Exactly_N_Messages_From_Topic("topic_B", TEST_CONSUMER, topicMsgCnt); + UNIT_ASSERT_VALUES_EQUAL(messages.front(), "message #1"); + UNIT_ASSERT_VALUES_EQUAL(messages.back(), "message #" + std::to_string(topicMsgCnt)); + } + + UNIT_ASSERT_VALUES_EQUAL(GetTableRecordsCount("table_A"), records.size()); + UNIT_ASSERT_VALUES_EQUAL(GetTableRecordsCount("table_B"), 0); + UNIT_ASSERT_VALUES_EQUAL(GetTableRecordsCount("table_C"), records.size()); + + CheckTabletKeys("topic_A"); + CheckTabletKeys("topic_B"); +} + +Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_4_Table, TFixtureSinksTable) +{ + TestSinksOlapWriteToTopicAndTable4(); +} + +Y_UNIT_TEST_F(Sinks_Olap_WriteToTopicAndTable_4_Query, TFixtureSinksQuery) +{ + TestSinksOlapWriteToTopicAndTable4(); +} + void TFixture::TestWriteRandomSizedMessagesInWideTransactions() { // The test verifies the simultaneous execution of several transactions. There is a topic From f5eea22cd44a36ad258a0528c4e9dce7037e6b4f Mon Sep 17 00:00:00 2001 From: Andrey Molotkov Date: Fri, 23 May 2025 14:27:48 +0000 Subject: [PATCH 05/16] Implement data erasure for PQ tablets (#17420) --- src/client/topic/ut/topic_to_table_ut.cpp | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/client/topic/ut/topic_to_table_ut.cpp b/src/client/topic/ut/topic_to_table_ut.cpp index 89deb6a7de..b5567bce83 100644 --- a/src/client/topic/ut/topic_to_table_ut.cpp +++ b/src/client/topic/ut/topic_to_table_ut.cpp @@ -355,7 +355,7 @@ class TFixture : public NUnitTest::TBaseFixture { void SplitPartition(const TString& topicPath, ui32 partitionId, const TString& boundary); - + virtual bool GetEnableOltpSink() const; virtual bool GetEnableOlapSink() const; virtual bool GetEnableHtapTx() const; @@ -1515,9 +1515,7 @@ ui64 TFixture::GetTopicTabletId(const TActorId& actorId, const TString& topicPat std::vector TFixture::GetTabletKeys(const TActorId& actorId, ui64 tabletId) { - using TEvKeyValue = NKikimr::TEvKeyValue; - - auto request = std::make_unique(); + auto request = std::make_unique(); request->Record.SetCookie(12345); auto cmd = request->Record.AddCmdReadRange(); @@ -1532,7 +1530,7 @@ std::vector TFixture::GetTabletKeys(const TActorId& actorId, auto& runtime = Setup->GetRuntime(); runtime.SendToPipe(tabletId, actorId, request.release()); - auto response = runtime.GrabEdgeEvent(); + auto response = runtime.GrabEdgeEvent(); UNIT_ASSERT(response->Record.HasCookie()); UNIT_ASSERT_VALUES_EQUAL(response->Record.GetCookie(), 12345); @@ -2211,16 +2209,14 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_10_Query, TFixtureQuery) NPQ::TWriteId TFixture::GetTransactionWriteId(const TActorId& actorId, ui64 tabletId) { - using TEvKeyValue = NKikimr::TEvKeyValue; - - auto request = std::make_unique(); + auto request = std::make_unique(); request->Record.SetCookie(12345); request->Record.AddCmdRead()->SetKey("_txinfo"); auto& runtime = Setup->GetRuntime(); runtime.SendToPipe(tabletId, actorId, request.release()); - auto response = runtime.GrabEdgeEvent(); + auto response = runtime.GrabEdgeEvent(); UNIT_ASSERT(response->Record.HasCookie()); UNIT_ASSERT_VALUES_EQUAL(response->Record.GetCookie(), 12345); @@ -2256,16 +2252,14 @@ void TFixture::WaitForTheTabletToDeleteTheWriteInfo(const TActorId& actorId, const NPQ::TWriteId& writeId) { while (true) { - using TEvKeyValue = NKikimr::TEvKeyValue; - - auto request = std::make_unique(); + auto request = std::make_unique(); request->Record.SetCookie(12345); request->Record.AddCmdRead()->SetKey("_txinfo"); auto& runtime = Setup->GetRuntime(); runtime.SendToPipe(tabletId, actorId, request.release()); - auto response = runtime.GrabEdgeEvent(); + auto response = runtime.GrabEdgeEvent(); UNIT_ASSERT(response->Record.HasCookie()); UNIT_ASSERT_VALUES_EQUAL(response->Record.GetCookie(), 12345); From c715192caa3081bb42c182f91635a10af83dd7da Mon Sep 17 00:00:00 2001 From: Daniil Cherednik Date: Fri, 23 May 2025 14:28:47 +0000 Subject: [PATCH 06/16] Return virtual timestamp for scan query (#14001) --- include/ydb-cpp-sdk/client/table/table.h | 11 ++++++++++- src/api/protos/ydb_query.proto | 2 ++ src/api/protos/ydb_table.proto | 2 ++ src/client/table/impl/readers.cpp | 9 ++++++++- 4 files changed, 22 insertions(+), 2 deletions(-) diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index cce1895cdf..8d88cc6b42 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -2063,6 +2063,8 @@ class TReadTableSnapshot { uint64_t TxId_; }; +using TVirtualTimestamp = TReadTableSnapshot; + template class TSimpleStreamPart : public TStreamPartStatus { public: @@ -2117,6 +2119,10 @@ class TScanQueryPart : public TStreamPartStatus { const std::string& GetDiagnostics() const { return *Diagnostics_; } std::string&& ExtractDiagnostics() { return std::move(*Diagnostics_); } + bool HasVirtualTimestamp() const { return Vt_.has_value(); } + const TVirtualTimestamp& GetVirtualTimestamp() const { return *Vt_; } + TVirtualTimestamp&& ExtractVirtualTimestamp() { return std::move(*Vt_); } + TScanQueryPart(TStatus&& status) : TStreamPartStatus(std::move(status)) {} @@ -2127,17 +2133,20 @@ class TScanQueryPart : public TStreamPartStatus { , Diagnostics_(diagnostics) {} - TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const std::optional& queryStats, const std::optional& diagnostics) + TScanQueryPart(TStatus&& status, TResultSet&& resultSet, const std::optional& queryStats, + const std::optional& diagnostics, std::optional&& vt) : TStreamPartStatus(std::move(status)) , ResultSet_(std::move(resultSet)) , QueryStats_(queryStats) , Diagnostics_(diagnostics) + , Vt_(std::move(vt)) {} private: std::optional ResultSet_; std::optional QueryStats_; std::optional Diagnostics_; + std::optional Vt_; }; using TAsyncScanQueryPart = NThreading::TFuture; diff --git a/src/api/protos/ydb_query.proto b/src/api/protos/ydb_query.proto index d3e61131cb..e1e4b8b698 100644 --- a/src/api/protos/ydb_query.proto +++ b/src/api/protos/ydb_query.proto @@ -8,6 +8,7 @@ option java_outer_classname = "QueryProtos"; import "google/protobuf/duration.proto"; import "src/api/protos/annotations/validation.proto"; +import "src/api/protos/ydb_common.proto"; import "src/api/protos/ydb_issue_message.proto"; import "src/api/protos/ydb_operation.proto"; import "src/api/protos/ydb_query_stats.proto"; @@ -200,6 +201,7 @@ message ExecuteQueryResponsePart { Ydb.TableStats.QueryStats exec_stats = 5; TransactionMeta tx_meta = 6; + VirtualTimestamp snapshot_timestamp = 7; } message ExecuteScriptRequest { diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index af2a6d869d..dff184d359 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -1293,6 +1293,8 @@ message ExecuteScanQueryPartialResult { // works only in mode: MODE_EXPLAIN, // collects additional diagnostics about query compilation, including query plan and scheme string query_full_diagnostics = 7 [deprecated = true]; + // Optional snapshot that corresponds to the returned data + VirtualTimestamp snapshot = 8; } // Returns information about an external data source with a given path. diff --git a/src/client/table/impl/readers.cpp b/src/client/table/impl/readers.cpp index 1d15c4d9aa..1ed0099cd7 100644 --- a/src/client/table/impl/readers.cpp +++ b/src/client/table/impl/readers.cpp @@ -89,9 +89,16 @@ TAsyncScanQueryPart TScanQueryPartIterator::TReaderImpl::ReadNext(std::shared_pt diagnostics = self->Response_.result().query_full_diagnostics(); + std::optional vt; + + if (self->Response_.result().has_snapshot()) { + const auto& snap = self->Response_.result().snapshot(); + vt = TVirtualTimestamp(snap.plan_step(), snap.tx_id()); + } + if (self->Response_.result().has_result_set()) { promise.SetValue({std::move(status), - TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats, diagnostics}); + TResultSet(std::move(*self->Response_.mutable_result()->mutable_result_set())), queryStats, diagnostics, std::move(vt)}); } else { promise.SetValue({std::move(status), queryStats, diagnostics}); } From 34861e27d2245efa2af4e8e2c39245f0c003e022 Mon Sep 17 00:00:00 2001 From: Evgeniy Ivanov Date: Fri, 23 May 2025 14:30:03 +0000 Subject: [PATCH 07/16] Allow to use per gRPC channel TCP connections (as an option) (#17857) --- include/ydb-cpp-sdk/client/driver/driver.h | 1 + src/client/driver/driver.cpp | 9 ++++++++- .../ydb_internal/grpc_connections/grpc_connections.cpp | 1 + .../ydb_internal/grpc_connections/grpc_connections.h | 5 +++++ src/client/impl/ydb_internal/grpc_connections/params.h | 1 + 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/include/ydb-cpp-sdk/client/driver/driver.h b/include/ydb-cpp-sdk/client/driver/driver.h index c934c422cb..60db7c097c 100644 --- a/include/ydb-cpp-sdk/client/driver/driver.h +++ b/include/ydb-cpp-sdk/client/driver/driver.h @@ -50,6 +50,7 @@ class TDriverConfig { //! caCerts - The buffer containing the PEM encoded root certificates for SSL/TLS connections. //! If this parameter is empty, the default roots will be used. TDriverConfig& UseSecureConnection(const std::string& caCerts = std::string()); + TDriverConfig& SetUsePerChannelTcpConnection(bool usePerChannel); TDriverConfig& UseClientCertificate(const std::string& clientCert, const std::string& clientPrivateKey); //! Set token, this option can be overridden for client by ClientSettings TDriverConfig& SetAuthToken(const std::string& token); diff --git a/src/client/driver/driver.cpp b/src/client/driver/driver.cpp index 7596a1840a..1016008bae 100644 --- a/src/client/driver/driver.cpp +++ b/src/client/driver/driver.cpp @@ -34,6 +34,7 @@ class TDriverConfig::TImpl : public IConnectionsParams { size_t GetClientThreadsNum() const override { return ClientThreadsNum; } size_t GetMaxQueuedResponses() const override { return MaxQueuedResponses; } TSslCredentials GetSslCredentials() const override { return SslCredentials; } + bool GetUsePerChannelTcpConnection() const override { return UsePerChannelTcpConnection; } std::string GetDatabase() const override { return Database; } std::shared_ptr GetCredentialsProviderFactory() const override { return CredentialsProviderFactory; } EDiscoveryMode GetDiscoveryMode() const override { return DiscoveryMode; } @@ -55,6 +56,7 @@ class TDriverConfig::TImpl : public IConnectionsParams { size_t ClientThreadsNum = 0; size_t MaxQueuedResponses = 0; TSslCredentials SslCredentials; + bool UsePerChannelTcpConnection = false; std::string Database; std::shared_ptr CredentialsProviderFactory = CreateInsecureCredentialsProviderFactory(); EDiscoveryMode DiscoveryMode = EDiscoveryMode::Sync; @@ -114,6 +116,11 @@ TDriverConfig& TDriverConfig::UseSecureConnection(const std::string& cert) { return *this; } +TDriverConfig& TDriverConfig::SetUsePerChannelTcpConnection(bool usePerChannel) { + Impl_->UsePerChannelTcpConnection = usePerChannel; + return *this; +} + TDriverConfig& TDriverConfig::UseClientCertificate(const std::string& clientCert, const std::string& clientPrivateKey) { Impl_->SslCredentials.Cert = clientCert; Impl_->SslCredentials.PrivateKey = clientPrivateKey; @@ -254,7 +261,7 @@ TDriverConfig TDriver::GetConfig() const { config.SetMaxOutboundMessageSize(Impl_->MaxOutboundMessageSize_); config.SetMaxMessageSize(Impl_->MaxMessageSize_); config.Impl_->Log = Impl_->Log; - + return config; } diff --git a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp index dc20d3a580..6004bdd5d9 100644 --- a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp +++ b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.cpp @@ -161,6 +161,7 @@ TGRpcConnectionsImpl::TGRpcConnectionsImpl(std::shared_ptr p , ChannelPool_(TcpKeepAliveSettings_, SocketIdleTimeout_) #endif , NetworkThreadsNum_(params->GetNetworkThreadsNum()) + , UsePerChannelTcpConnection_(params->GetUsePerChannelTcpConnection()) , GRpcClientLow_(NetworkThreadsNum_) , Log(params->GetLog()) { diff --git a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h index db777210ca..e6658783bf 100644 --- a/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h +++ b/src/client/impl/ydb_internal/grpc_connections/grpc_connections.h @@ -117,6 +117,10 @@ class TGRpcConnectionsImpl } } + if (UsePerChannelTcpConnection_) { + clientConfig.IntChannelParams[GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL] = 1; + } + std::unique_ptr> conn; #ifndef YDB_GRPC_BYPASS_CHANNEL_POOL ChannelPool_.GetStubsHolderLocked( @@ -733,6 +737,7 @@ class TGRpcConnectionsImpl IDiscoveryMutatorApi::TMutatorCb DiscoveryMutatorCb; const size_t NetworkThreadsNum_; + bool UsePerChannelTcpConnection_; // Must be the last member (first called destructor) NYdbGrpc::TGRpcClientLow GRpcClientLow_; TLog Log; diff --git a/src/client/impl/ydb_internal/grpc_connections/params.h b/src/client/impl/ydb_internal/grpc_connections/params.h index b0670a8e48..d03c7258f9 100644 --- a/src/client/impl/ydb_internal/grpc_connections/params.h +++ b/src/client/impl/ydb_internal/grpc_connections/params.h @@ -17,6 +17,7 @@ class IConnectionsParams { virtual size_t GetClientThreadsNum() const = 0; virtual size_t GetMaxQueuedResponses() const = 0; virtual TSslCredentials GetSslCredentials() const = 0; + virtual bool GetUsePerChannelTcpConnection() const = 0; virtual std::string GetDatabase() const = 0; virtual std::shared_ptr GetCredentialsProviderFactory() const = 0; virtual EDiscoveryMode GetDiscoveryMode() const = 0; From c4994f10a4d796855d933a6c4de89b854628c71a Mon Sep 17 00:00:00 2001 From: Stanislav <25842793+raydzast@users.noreply.github.com> Date: Fri, 23 May 2025 14:31:08 +0000 Subject: [PATCH 08/16] Followers support for secondary indexes (#17965) --- include/ydb-cpp-sdk/client/table/table.h | 41 +++++++------ src/api/protos/ydb_table.proto | 1 + src/client/table/table.cpp | 77 ++++++++++++------------ 3 files changed, 61 insertions(+), 58 deletions(-) diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index 8d88cc6b42..134688ffd0 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -31,6 +31,7 @@ class GlobalIndexSettings; class VectorIndexSettings; class KMeansTreeSettings; class PartitioningSettings; +class ReadReplicasSettings; class DateTypeColumnModeSettings; class TtlSettings; class TtlTier; @@ -200,11 +201,33 @@ struct TExplicitPartitions { void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const; }; +//! Represents table read replicas settings +class TReadReplicasSettings { +public: + enum class EMode { + PerAz = 0, + AnyAz = 1 + }; + + TReadReplicasSettings(EMode mode, uint64_t readReplicasCount); + + EMode GetMode() const; + uint64_t GetReadReplicasCount() const; + + static std::optional FromProto(const Ydb::Table::ReadReplicasSettings& proto); + void SerializeTo(Ydb::Table::ReadReplicasSettings& proto) const; + +private: + EMode Mode_; + uint64_t ReadReplicasCount_; +}; + struct TGlobalIndexSettings { using TUniformOrExplicitPartitions = std::variant; TPartitioningSettings PartitioningSettings; TUniformOrExplicitPartitions Partitions; + std::optional ReadReplicasSettings; static TGlobalIndexSettings FromProto(const Ydb::Table::GlobalIndexSettings& proto); @@ -630,24 +653,6 @@ class TColumnFamilyDescription { std::shared_ptr Impl_; }; -//! Represents table read replicas settings -class TReadReplicasSettings { -public: - enum class EMode { - PerAz = 0, - AnyAz = 1 - }; - - TReadReplicasSettings(EMode mode, uint64_t readReplicasCount); - - EMode GetMode() const; - uint64_t GetReadReplicasCount() const; - -private: - EMode Mode_; - uint64_t ReadReplicasCount_; -}; - enum class EStoreType { Row = 0, Column = 1 diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index dff184d359..270a91ad1f 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -58,6 +58,7 @@ message GlobalIndexSettings { } // Partitioning settings for the table that implements the index. PartitioningSettings partitioning_settings = 3; + ReadReplicasSettings read_replicas_settings = 4; } message VectorIndexSettings { diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 37810f48f7..8829f16c45 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -359,23 +359,7 @@ class TTableDescription::TImpl { } // read replicas settings - if (proto.has_read_replicas_settings()) { - const auto settings = proto.read_replicas_settings(); - switch (settings.settings_case()) { - case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: - ReadReplicasSettings_ = TReadReplicasSettings( - TReadReplicasSettings::EMode::PerAz, - settings.per_az_read_replicas_count()); - break; - case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: - ReadReplicasSettings_ = TReadReplicasSettings( - TReadReplicasSettings::EMode::AnyAz, - settings.any_az_read_replicas_count()); - break; - default: - break; - } - } + ReadReplicasSettings_ = TReadReplicasSettings::FromProto(proto.read_replicas_settings()); } public: @@ -973,16 +957,7 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con } if (const auto& settings = Impl_->GetReadReplicasSettings()) { - switch (settings->GetMode()) { - case TReadReplicasSettings::EMode::PerAz: - request.mutable_read_replicas_settings()->set_per_az_read_replicas_count(settings->GetReadReplicasCount()); - break; - case TReadReplicasSettings::EMode::AnyAz: - request.mutable_read_replicas_settings()->set_any_az_read_replicas_count(settings->GetReadReplicasCount()); - break; - default: - break; - } + settings->SerializeTo(*request.mutable_read_replicas_settings()); } } @@ -1745,18 +1720,7 @@ static Ydb::Table::AlterTableRequest MakeAlterTableProtoRequest( if (settings.SetReadReplicasSettings_.has_value()) { const auto& replSettings = settings.SetReadReplicasSettings_.value(); - switch (replSettings.GetMode()) { - case TReadReplicasSettings::EMode::PerAz: - request.mutable_set_read_replicas_settings()->set_per_az_read_replicas_count( - replSettings.GetReadReplicasCount()); - break; - case TReadReplicasSettings::EMode::AnyAz: - request.mutable_set_read_replicas_settings()->set_any_az_read_replicas_count( - replSettings.GetReadReplicasCount()); - break; - default: - break; - } + replSettings.SerializeTo(*request.mutable_set_read_replicas_settings()); } return request; @@ -2373,6 +2337,34 @@ uint64_t TIndexDescription::GetSizeBytes() const { return SizeBytes_; } +std::optional TReadReplicasSettings::FromProto(const Ydb::Table::ReadReplicasSettings& proto) { + switch (proto.settings_case()) { + case Ydb::Table::ReadReplicasSettings::kPerAzReadReplicasCount: + return TReadReplicasSettings( + TReadReplicasSettings::EMode::PerAz, + proto.per_az_read_replicas_count()); + case Ydb::Table::ReadReplicasSettings::kAnyAzReadReplicasCount: + return TReadReplicasSettings( + TReadReplicasSettings::EMode::AnyAz, + proto.any_az_read_replicas_count()); + default: + return { }; + } +} + +void TReadReplicasSettings::SerializeTo(Ydb::Table::ReadReplicasSettings& proto) const { + switch (GetMode()) { + case TReadReplicasSettings::EMode::PerAz: + proto.set_per_az_read_replicas_count(GetReadReplicasCount()); + break; + case TReadReplicasSettings::EMode::AnyAz: + proto.set_any_az_read_replicas_count(GetReadReplicasCount()); + break; + default: + break; + } +} + TGlobalIndexSettings TGlobalIndexSettings::FromProto(const Ydb::Table::GlobalIndexSettings& proto) { auto partitionsFromProto = [](const Ydb::Table::GlobalIndexSettings& proto) -> TUniformOrExplicitPartitions { switch (proto.partitions_case()) { @@ -2387,7 +2379,8 @@ TGlobalIndexSettings TGlobalIndexSettings::FromProto(const Ydb::Table::GlobalInd return { .PartitioningSettings = TPartitioningSettings(proto.partitioning_settings()), - .Partitions = partitionsFromProto(proto) + .Partitions = partitionsFromProto(proto), + .ReadReplicasSettings = TReadReplicasSettings::FromProto(proto.read_replicas_settings()) }; } @@ -2403,6 +2396,10 @@ void TGlobalIndexSettings::SerializeTo(Ydb::Table::GlobalIndexSettings& settings } }; std::visit(std::move(variantVisitor), Partitions); + + if (ReadReplicasSettings) { + ReadReplicasSettings->SerializeTo(*settings.mutable_read_replicas_settings()); + } } TVectorIndexSettings TVectorIndexSettings::FromProto(const Ydb::Table::VectorIndexSettings& proto) { From 884bf0012391b3bec258a87083d7fd14be15e5a7 Mon Sep 17 00:00:00 2001 From: azevaykin <145343289+azevaykin@users.noreply.github.com> Date: Fri, 23 May 2025 14:32:10 +0000 Subject: [PATCH 09/16] Prefix vector impl table has its own settings (#18132) --- src/api/protos/ydb_table.proto | 1 + src/client/table/table.cpp | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/api/protos/ydb_table.proto b/src/api/protos/ydb_table.proto index 270a91ad1f..2ab29249c9 100644 --- a/src/api/protos/ydb_table.proto +++ b/src/api/protos/ydb_table.proto @@ -109,6 +109,7 @@ message GlobalUniqueIndex { message GlobalVectorKMeansTreeIndex { GlobalIndexSettings level_table_settings = 1; GlobalIndexSettings posting_table_settings = 2; + GlobalIndexSettings prefix_table_settings = 4; KMeansTreeSettings vector_settings = 3; } diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index 8829f16c45..0c3c93ee53 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -2531,6 +2531,10 @@ TIndexDescription TIndexDescription::FromProto(const TProto& proto) { const auto &vectorProto = proto.global_vector_kmeans_tree_index(); globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(vectorProto.level_table_settings())); globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(vectorProto.posting_table_settings())); + const bool prefixVectorIndex = indexColumns.size() > 1; + if (prefixVectorIndex) { + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(vectorProto.prefix_table_settings())); + } specializedIndexSettings = TKMeansTreeSettings::FromProto(vectorProto.vector_settings()); break; } From 20e71bad65dc81a00491665892f5edf1d3502fa4 Mon Sep 17 00:00:00 2001 From: Vitaliy Filippov Date: Fri, 23 May 2025 14:32:52 +0000 Subject: [PATCH 10/16] Add StartTime/EndTime/UserSID for BuildIndex (#16971) (#17751) --- src/api/protos/ydb_topic.proto | 4 ++++ src/client/types/operation/operation.cpp | 1 + 2 files changed, 5 insertions(+) diff --git a/src/api/protos/ydb_topic.proto b/src/api/protos/ydb_topic.proto index e28e0bce42..5e0f65838f 100644 --- a/src/api/protos/ydb_topic.proto +++ b/src/api/protos/ydb_topic.proto @@ -688,6 +688,10 @@ message StreamDirectReadMessage { // Messages data StreamReadMessage.ReadResponse.PartitionData partition_data = 3; + + // Total size in bytes of this response as calculated by server. + // See ReadRequest comment above. + int64 bytes_size = 4; } } diff --git a/src/client/types/operation/operation.cpp b/src/client/types/operation/operation.cpp index 975cdb8891..b32d047424 100644 --- a/src/client/types/operation/operation.cpp +++ b/src/client/types/operation/operation.cpp @@ -23,6 +23,7 @@ class TOperation::TImpl { , Ready_(operation.ready()) , CreateTime_(ProtoTimestampToInstant(operation.create_time())) , EndTime_(ProtoTimestampToInstant(operation.end_time())) + , CreatedBy_(operation.created_by()) , Operation_(std::move(operation)) { } From 4042262097440549dfa790eacbc0a2933648a576 Mon Sep 17 00:00:00 2001 From: Alexander Rutkovsky Date: Fri, 23 May 2025 14:34:22 +0000 Subject: [PATCH 11/16] Allow fetching extra sections through distconf to revert automatic configuration changes (#18107) --- src/api/protos/ydb_config.proto | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/api/protos/ydb_config.proto b/src/api/protos/ydb_config.proto index aae8a06e7b..3c6af15e85 100644 --- a/src/api/protos/ydb_config.proto +++ b/src/api/protos/ydb_config.proto @@ -114,7 +114,8 @@ message FetchConfigRequest { message FetchModeAll { // Use this option to somehow transform main config // Currently used to automatically derive detached StorageSection from MainConfig - // or vice versa + // or vice versa, and to add some internally-managed sections into config for + // explicit user control oneof config_transform { // Optionally may be set to explicitly tell "do not transform anything" google.protobuf.Empty none = 1; @@ -130,6 +131,13 @@ message FetchConfigRequest { // (MainConfig*, StorageConfig) -> (MainConfig) // MainConfig with asterisk means MainConfig with excluded storage-related sections google.protobuf.Empty attach_storage_config_section = 3; + + // Fetch will return MainConfig with added blob_storage_config and domains_config sections in order to + // downgrade to configuration v1. + google.protobuf.Empty add_blob_storage_and_domains_config = 4; + + // Fetch will return MainConfig/StorageConfig with added blob_storage_config and explicit_* sections + google.protobuf.Empty add_explicit_sections = 5; } } From 0036ea231f9139b030ca5520a8e9718f322b5908 Mon Sep 17 00:00:00 2001 From: Vasily Gerasimov Date: Fri, 23 May 2025 14:35:18 +0000 Subject: [PATCH 12/16] TImportClient::ListObjectsInS3Export in C++ SDK (#18366) --- include/ydb-cpp-sdk/client/import/import.h | 56 ++++++++- src/api/grpc/ydb_import_v1.proto | 3 + src/api/protos/ydb_import.proto | 81 +++++++++++++ src/client/import/import.cpp | 128 +++++++++++++++++---- src/client/import/out.cpp | 12 ++ 5 files changed, 253 insertions(+), 27 deletions(-) diff --git a/include/ydb-cpp-sdk/client/import/import.h b/include/ydb-cpp-sdk/client/import/import.h index 96ccfde817..809cf520c1 100644 --- a/include/ydb-cpp-sdk/client/import/import.h +++ b/include/ydb-cpp-sdk/client/import/import.h @@ -5,6 +5,10 @@ #include +namespace Ydb::Import { +class ListObjectsInS3ExportResult; +} + namespace NYdb::inline V3 { namespace NImport { @@ -77,6 +81,49 @@ class TImportFromS3Response : public TOperation { TMetadata Metadata_; }; +using TAsyncImportFromS3Response = NThreading::TFuture; + +struct TListObjectsInS3ExportSettings : public TOperationRequestSettings, + public TS3Settings { + using TSelf = TListObjectsInS3ExportSettings; + + struct TItem { + // Database object path. + std::string Path = {}; + }; + + FLUENT_SETTING_VECTOR(TItem, Item); + FLUENT_SETTING_OPTIONAL(uint32_t, NumberOfRetries); + FLUENT_SETTING_OPTIONAL(std::string, Prefix); + FLUENT_SETTING_OPTIONAL(std::string, SymmetricKey); +}; + +class TListObjectsInS3ExportResult : public TStatus { +public: + struct TItem { + // S3 object prefix + std::string Prefix; + + // Database object path + std::string Path; + + void Out(IOutputStream& out) const; + }; + + TListObjectsInS3ExportResult(TStatus&& status, const ::Ydb::Import::ListObjectsInS3ExportResult& proto); + + const std::vector& GetItems() const; + const std::string& NextPageToken() const { return NextPageToken_; } + + void Out(IOutputStream& out) const; + +private: + std::vector Items_; + std::string NextPageToken_; +}; + +using TAsyncListObjectsInS3ExportResult = NThreading::TFuture; + /// Data struct TImportYdbDumpDataSettings : public TOperationRequestSettings { using TSelf = TImportYdbDumpDataSettings; @@ -99,7 +146,9 @@ class TImportClient { public: TImportClient(const TDriver& driver, const TCommonClientSettings& settings = TCommonClientSettings()); - NThreading::TFuture ImportFromS3(const TImportFromS3Settings& settings); + TAsyncImportFromS3Response ImportFromS3(const TImportFromS3Settings& settings); + + TAsyncListObjectsInS3ExportResult ListObjectsInS3Export(const TListObjectsInS3ExportSettings& settings, std::int64_t pageSize = 0, const std::string& pageToken = {}); // ydb dump format TAsyncImportDataResult ImportData(const std::string& table, std::string&& data, const TImportYdbDumpDataSettings& settings); @@ -111,8 +160,3 @@ class TImportClient { } // namespace NImport } // namespace NYdb - -template<> -inline void Out(IOutputStream& o, const NYdb::NImport::TImportFromS3Response& x) { - return x.Out(o); -} diff --git a/src/api/grpc/ydb_import_v1.proto b/src/api/grpc/ydb_import_v1.proto index 1fec2cf96c..4803421869 100644 --- a/src/api/grpc/ydb_import_v1.proto +++ b/src/api/grpc/ydb_import_v1.proto @@ -11,6 +11,9 @@ service ImportService { // Method starts an asynchronous operation that can be cancelled while it is in progress. rpc ImportFromS3(Import.ImportFromS3Request) returns (Import.ImportFromS3Response); + // List objects from existing export stored in S3 bucket + rpc ListObjectsInS3Export(Import.ListObjectsInS3ExportRequest) returns (Import.ListObjectsInS3ExportResponse); + // Writes data to a table. // Method accepts serialized data in the selected format and writes it non-transactionally. rpc ImportData(Import.ImportDataRequest) returns (Import.ImportDataResponse); diff --git a/src/api/protos/ydb_import.proto b/src/api/protos/ydb_import.proto index 3e21023525..50e48dbba7 100644 --- a/src/api/protos/ydb_import.proto +++ b/src/api/protos/ydb_import.proto @@ -121,6 +121,87 @@ message ImportFromS3Response { Ydb.Operations.Operation operation = 1; } +message ListObjectsInS3ExportSettings { + message Item { + // Database object path + // Recursive for directories + string path = 1; + } + + string endpoint = 1 [(required) = true]; + ImportFromS3Settings.Scheme scheme = 2; // HTTPS if not specified + string bucket = 3 [(required) = true]; + string access_key = 4 [(required) = true]; + string secret_key = 5 [(required) = true]; + repeated Item items = 6; + uint32 number_of_retries = 7; + + // Region to use in requests + string region = 8; + + // disables virtual hosting style buckets aws s3 feature + // it changes the way bucket appended to url. e.g. https//bucket_name.example.com/ vs https://example.com/bucket_name + // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html + // it is especially useful for custom s3 implementations + bool disable_virtual_addressing = 9; + + // A default path prefix for all items, + // determines that the import works with the list of objects in the SchemaMapping file. + // Must be provided for encrypted exports. + string prefix = 10; + + // Settings how data is encrypted. + // If encryption_settings field is not specified, + // the resulting data is considered not encrypted. + Ydb.Export.EncryptionSettings encryption_settings = 11; +} + +message ListObjectsInS3ExportResult { + message Item { + /* YDB database objects in S3 are stored in one or more S3 objects (see ydb_export.proto). + The S3 object name begins with a prefix, followed by: + * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero; + * '/scheme.pb' - object with information about scheme, indexes, etc; + * '/permissions.pb' - object with information about ACL and owner. + */ + + // S3 object prefix + string prefix = 1; + + // Database object path + string path = 2; + } + + repeated Item items = 1; + + // This token allows you to get the next page of results for ListObjectsInS3Export requests, + // if the number of results is larger than `page_size` specified in the request. + // To get the next page, specify the value of `next_page_token` as a value for + // the `page_token` parameter in the next ListObjectsInS3Export request. Subsequent ListObjectsInS3Export + // requests will have their own `next_page_token` to continue paging through the results. + string next_page_token = 2; +} + +message ListObjectsInS3ExportRequest { + Ydb.Operations.OperationParams operation_params = 1; + ListObjectsInS3ExportSettings settings = 2 [(required) = true]; + + // The maximum number of results per page that should be returned. If the number of available + // results is larger than `page_size`, the service returns a `next_page_token` that can be used + // to get the next page of results in subsequent ListObjectsInS3Export requests. + // 0 means that server returns all objects. + int64 page_size = 3 [(value) = "<= 10000"]; + + // Page token. Set `page_token` to the `next_page_token` returned by a previous ListObjectsInS3Export + // request to get the next page of results. + string page_token = 4; +} + +message ListObjectsInS3ExportResponse { + // operation.result = ListObjectsInS3ExportResult + Ydb.Operations.Operation operation = 1; +} + /// Data message YdbDumpFormat { repeated string columns = 1; diff --git a/src/client/import/import.cpp b/src/client/import/import.cpp index 506841959d..f6c62f43f6 100644 --- a/src/client/import/import.cpp +++ b/src/client/import/import.cpp @@ -10,6 +10,8 @@ #include #include +#include + namespace NYdb::inline V3 { namespace NImport { @@ -34,6 +36,25 @@ std::vector ItemsProgressFromProto(const google::protobuf:: return result; } +template +void FillS3Settings(TS3SettingsProto& proto, const TSettings& settings) { + proto.set_endpoint(TStringType{settings.Endpoint_}); + proto.set_scheme(TProtoAccessor::GetProto(settings.Scheme_)); + proto.set_bucket(TStringType{settings.Bucket_}); + proto.set_access_key(TStringType{settings.AccessKey_}); + proto.set_secret_key(TStringType{settings.SecretKey_}); + + if (settings.NumberOfRetries_) { + proto.set_number_of_retries(settings.NumberOfRetries_.value()); + } + + if (settings.SymmetricKey_) { + proto.mutable_encryption_settings()->mutable_symmetric_key()->set_key(*settings.SymmetricKey_); + } + + proto.set_disable_virtual_addressing(!settings.UseVirtualAddressing_); +} + } // anonymous /// S3 @@ -66,6 +87,36 @@ const TImportFromS3Response::TMetadata& TImportFromS3Response::Metadata() const return Metadata_; } +TListObjectsInS3ExportResult::TListObjectsInS3ExportResult(TStatus&& status, const ::Ydb::Import::ListObjectsInS3ExportResult& proto) + : TStatus(std::move(status)) +{ + Items_.reserve(proto.items_size()); + for (const auto& item : proto.items()) { + Items_.emplace_back(TItem{ + .Prefix = item.prefix(), + .Path = item.path() + }); + } + NextPageToken_ = proto.next_page_token(); +} + +const std::vector& TListObjectsInS3ExportResult::GetItems() const { + return Items_; +} + +void TListObjectsInS3ExportResult::Out(IOutputStream& out) const { + if (IsSuccess()) { + out << "{ items: [" << JoinSeq(", ", Items_) << "], next_page_token: \"" << NextPageToken_ << "\" }"; + } else { + return TStatus::Out(out); + } +} + +void TListObjectsInS3ExportResult::TItem::Out(IOutputStream& out) const { + out << "{ prefix: \"" << Prefix << "\"" + << ", path: \"" << Path << "\" }"; +} + /// Data TImportDataResult::TImportDataResult(TStatus&& status) : TStatus(std::move(status)) @@ -80,13 +131,37 @@ class TImportClient::TImpl : public TClientImplCommon { { } - TFuture ImportFromS3(ImportFromS3Request&& request, const TImportFromS3Settings& settings) { + TAsyncImportFromS3Response ImportFromS3(ImportFromS3Request&& request, const TImportFromS3Settings& settings) { return RunOperation( std::move(request), &V1::ImportService::Stub::AsyncImportFromS3, TRpcRequestSettings::Make(settings)); } + TAsyncListObjectsInS3ExportResult ListObjectsInS3Export(ListObjectsInS3ExportRequest&& request, const TListObjectsInS3ExportSettings& settings) { + auto promise = NThreading::NewPromise(); + + auto extractor = [promise] + (google::protobuf::Any* any, TPlainStatus status) mutable { + ListObjectsInS3ExportResult result; + if (any) { + any->UnpackTo(&result); + } + + promise.SetValue(TListObjectsInS3ExportResult(TStatus(std::move(status)), result)); + }; + + Connections_->RunDeferred( + std::move(request), + extractor, + &V1::ImportService::Stub::AsyncListObjectsInS3Export, + DbDriverState_, + INITIAL_DEFERRED_CALL_DELAY, + TRpcRequestSettings::Make(settings)); + + return promise.GetFuture(); + } + template TAsyncImportDataResult ImportData(ImportDataRequest&& request, const TSettings& settings) { auto promise = NThreading::NewPromise(); @@ -131,14 +206,10 @@ TImportClient::TImportClient(const TDriver& driver, const TCommonClientSettings& { } -TFuture TImportClient::ImportFromS3(const TImportFromS3Settings& settings) { +TAsyncImportFromS3Response TImportClient::ImportFromS3(const TImportFromS3Settings& settings) { auto request = MakeOperationRequest(settings); - - request.mutable_settings()->set_endpoint(TStringType{settings.Endpoint_}); - request.mutable_settings()->set_scheme(TProtoAccessor::GetProto(settings.Scheme_)); - request.mutable_settings()->set_bucket(TStringType{settings.Bucket_}); - request.mutable_settings()->set_access_key(TStringType{settings.AccessKey_}); - request.mutable_settings()->set_secret_key(TStringType{settings.SecretKey_}); + Ydb::Import::ImportFromS3Settings& settingsProto = *request.mutable_settings(); + FillS3Settings(settingsProto, settings); for (const auto& item : settings.Item_) { if (!item.Src.empty() && !item.SrcPath.empty()) { @@ -146,7 +217,7 @@ TFuture TImportClient::ImportFromS3(const TImportFromS3Se TStringBuilder() << "Invalid item: both source prefix and source path are set: \"" << item.Src << "\" and \"" << item.SrcPath << "\""); } - auto& protoItem = *request.mutable_settings()->mutable_items()->Add(); + auto& protoItem = *settingsProto.mutable_items()->Add(); if (!item.Src.empty()) { protoItem.set_source_prefix(item.Src); } @@ -157,32 +228,47 @@ TFuture TImportClient::ImportFromS3(const TImportFromS3Se } if (settings.Description_) { - request.mutable_settings()->set_description(TStringType{settings.Description_.value()}); - } - - if (settings.NumberOfRetries_) { - request.mutable_settings()->set_number_of_retries(settings.NumberOfRetries_.value()); + settingsProto.set_description(TStringType{settings.Description_.value()}); } if (settings.NoACL_) { - request.mutable_settings()->set_no_acl(settings.NoACL_.value()); + settingsProto.set_no_acl(settings.NoACL_.value()); } if (settings.SourcePrefix_) { - request.mutable_settings()->set_source_prefix(settings.SourcePrefix_.value()); + settingsProto.set_source_prefix(settings.SourcePrefix_.value()); } if (settings.DestinationPath_) { - request.mutable_settings()->set_destination_path(settings.DestinationPath_.value()); + settingsProto.set_destination_path(settings.DestinationPath_.value()); } - if (settings.SymmetricKey_) { - request.mutable_settings()->mutable_encryption_settings()->mutable_symmetric_key()->set_key(*settings.SymmetricKey_); + return Impl_->ImportFromS3(std::move(request), settings); +} + +TAsyncListObjectsInS3ExportResult TImportClient::ListObjectsInS3Export(const TListObjectsInS3ExportSettings& settings, std::int64_t pageSize, const std::string& pageToken) { + auto request = MakeOperationRequest(settings); + Ydb::Import::ListObjectsInS3ExportSettings& settingsProto = *request.mutable_settings(); + FillS3Settings(settingsProto, settings); + + if (settings.Prefix_) { + settingsProto.set_prefix(settings.Prefix_.value()); } - request.mutable_settings()->set_disable_virtual_addressing(!settings.UseVirtualAddressing_); + for (const auto& item : settings.Item_) { + if (item.Path.empty()) { + throw TContractViolation( + TStringBuilder() << "Invalid item: path is not set"); + } - return Impl_->ImportFromS3(std::move(request), settings); + settingsProto.add_items()->set_path(item.Path); + } + + // Paging + request.set_page_size(pageSize); + request.set_page_token(pageToken); + + return Impl_->ListObjectsInS3Export(std::move(request), settings); } TAsyncImportDataResult TImportClient::ImportData(const std::string& table, std::string&& data, const TImportYdbDumpDataSettings& settings) { diff --git a/src/client/import/out.cpp b/src/client/import/out.cpp index af523528dd..8828336c11 100644 --- a/src/client/import/out.cpp +++ b/src/client/import/out.cpp @@ -3,3 +3,15 @@ Y_DECLARE_OUT_SPEC(, NYdb::NImport::TImportDataResult, o, x) { return x.Out(o); } + +Y_DECLARE_OUT_SPEC(, NYdb::NImport::TImportFromS3Response, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NImport::TListObjectsInS3ExportResult, o, x) { + return x.Out(o); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NImport::TListObjectsInS3ExportResult::TItem, o, x) { + return x.Out(o); +} From d876f901ba42620757d122503114dcab33db858f Mon Sep 17 00:00:00 2001 From: Pisarenko Grigoriy Date: Fri, 23 May 2025 14:36:22 +0000 Subject: [PATCH 13/16] Fixed operation id ProtoToString (#18460) --- src/library/operation_id/operation_id.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/library/operation_id/operation_id.cpp b/src/library/operation_id/operation_id.cpp index 1a24ff493b..3a62e66ed9 100644 --- a/src/library/operation_id/operation_id.cpp +++ b/src/library/operation_id/operation_id.cpp @@ -39,6 +39,8 @@ std::string ProtoToString(const Ydb::TOperationId& proto) { reflection.ListFields(proto, &fields); TStringStream res; switch (proto.kind()) { + case Ydb::TOperationId::UNUSED: + break; case Ydb::TOperationId::OPERATION_DDL: case Ydb::TOperationId::OPERATION_DML: res << "ydb://operation"; From 815a4bf3d361545037e3f9b63deb94674683081e Mon Sep 17 00:00:00 2001 From: Dmitry Raspopov Date: Fri, 23 May 2025 14:37:13 +0000 Subject: [PATCH 14/16] bug ydb-cpp-sdk: fix federated topic debug print templates (#18502) --- .../client/federated_topic/federated_topic.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h b/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h index ab2ca5bbb6..5bf3f4c4ed 100644 --- a/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h +++ b/include/ydb-cpp-sdk/client/federated_topic/federated_topic.h @@ -564,20 +564,20 @@ void TPrintable::DebugString(TStringBuilder& res, bo template<> void TPrintable::DebugString(TStringBuilder& res, bool) const; template<> -void TPrintable>::DebugString(TStringBuilder& res, bool) const; +void TPrintable::DebugString(TStringBuilder& res, bool) const; template<> -void TPrintable>::DebugString(TStringBuilder& res, bool) const; +void TPrintable::DebugString(TStringBuilder& res, bool) const; template<> -void TPrintable>::DebugString(TStringBuilder& res, bool) const; +void TPrintable::DebugString(TStringBuilder& res, bool) const; template<> -void TPrintable>::DebugString(TStringBuilder& res, bool) const; +void TPrintable::DebugString(TStringBuilder& res, bool) const; template<> -void TPrintable>::DebugString(TStringBuilder& res, bool) const; +void TPrintable::DebugString(TStringBuilder& res, bool) const; template<> -void TPrintable>::DebugString(TStringBuilder& res, bool) const; +void TPrintable::DebugString(TStringBuilder& res, bool) const; template<> -void TPrintable>::DebugString(TStringBuilder& res, bool) const; +void TPrintable::DebugString(TStringBuilder& res, bool) const; template<> -void TPrintable>::DebugString(TStringBuilder& res, bool) const; +void TPrintable::DebugString(TStringBuilder& res, bool) const; } From 6542ce55ff84e42618e3d2bbdc5f95b9d79094d9 Mon Sep 17 00:00:00 2001 From: Bulat Date: Fri, 23 May 2025 14:38:12 +0000 Subject: [PATCH 15/16] [C++ SDK] Fixed bug in TQueryClient::Commit (#18701) --- src/client/query/client.cpp | 10 +++++++--- src/client/table/impl/transaction.cpp | 8 ++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 4f5c7622ec..7e446e8b1a 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -752,9 +752,11 @@ class TTransaction::TImpl : public std::enable_shared_from_this { } TAsyncStatus Precommit() const { + auto self = shared_from_this(); + TStatus status(EStatus::SUCCESS, {}); - for (auto& callback : PrecommitCallbacks) { + for (auto& callback : self->PrecommitCallbacks) { if (!callback) { continue; } @@ -775,7 +777,9 @@ class TTransaction::TImpl : public std::enable_shared_from_this { } NThreading::TFuture ProcessFailure() const { - for (auto& callback : OnFailureCallbacks) { + auto self = shared_from_this(); + + for (auto& callback : self->OnFailureCallbacks) { if (!callback) { continue; } @@ -803,7 +807,7 @@ class TTransaction::TImpl : public std::enable_shared_from_this { co_return TCommitTransactionResult(TStatus(precommitResult)); } - PrecommitCallbacks.clear(); + self->PrecommitCallbacks.clear(); auto commitResult = co_await self->Session_.Client_->CommitTransaction(self->TxId_, settingsCopy, self->Session_); diff --git a/src/client/table/impl/transaction.cpp b/src/client/table/impl/transaction.cpp index 0c0340abb1..ecf4c07251 100644 --- a/src/client/table/impl/transaction.cpp +++ b/src/client/table/impl/transaction.cpp @@ -11,9 +11,11 @@ TTransaction::TImpl::TImpl(const TSession& session, const std::string& txId) TAsyncStatus TTransaction::TImpl::Precommit() const { + auto self = shared_from_this(); + TStatus status(EStatus::SUCCESS, {}); - for (auto& callback : PrecommitCallbacks) { + for (auto& callback : self->PrecommitCallbacks) { if (!callback) { continue; } @@ -35,7 +37,9 @@ TAsyncStatus TTransaction::TImpl::Precommit() const NThreading::TFuture TTransaction::TImpl::ProcessFailure() const { - for (auto& callback : OnFailureCallbacks) { + auto self = shared_from_this(); + + for (auto& callback : self->OnFailureCallbacks) { if (!callback) { continue; } From 9546e400776fe1fae03b84683d8f2e4bced0e0f9 Mon Sep 17 00:00:00 2001 From: Bulat Date: Fri, 23 May 2025 14:42:20 +0000 Subject: [PATCH 16/16] [C++ SDK] Adapted basic usage topic test for ydb-cpp-sdk repo (#18707) --- .github/workflows/tests.yaml | 2 +- src/client/persqueue_public/CMakeLists.txt | 27 +- .../persqueue_public/impl/CMakeLists.txt | 1 + .../persqueue_public/impl/persqueue.cpp | 2 +- .../persqueue_public/include/CMakeLists.txt | 32 + src/client/topic/impl/read_session_impl.ipp | 10 +- src/client/topic/ut/basic_usage_ut.cpp | 826 ------------- tests/integration/CMakeLists.txt | 1 + tests/integration/topic/CMakeLists.txt | 11 + tests/integration/topic/basic_usage.cpp | 1063 +++++++++++++++++ 10 files changed, 1117 insertions(+), 858 deletions(-) create mode 100644 src/client/persqueue_public/include/CMakeLists.txt create mode 100644 tests/integration/topic/CMakeLists.txt create mode 100644 tests/integration/topic/basic_usage.cpp diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 93b3cb98fc..1ba010d6d6 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -116,4 +116,4 @@ jobs: - name: Test shell: bash run: | - ctest -j$(nproc) --preset integration + YDB_VERSION=${{ matrix.ydb-version }} ctest -j$(nproc) --preset integration diff --git a/src/client/persqueue_public/CMakeLists.txt b/src/client/persqueue_public/CMakeLists.txt index d7ad2dfca9..604a1f75e2 100644 --- a/src/client/persqueue_public/CMakeLists.txt +++ b/src/client/persqueue_public/CMakeLists.txt @@ -1,35 +1,12 @@ add_subdirectory(codecs) add_subdirectory(impl) +add_subdirectory(include) _ydb_sdk_add_library(cpp-client-ydb_persqueue_public INTERFACE) target_link_libraries(cpp-client-ydb_persqueue_public INTERFACE yutil - cpp-client-ydb_persqueue_core + client-ydb_persqueue_public-include client-ydb_persqueue_public-impl client-ydb_persqueue_public-codecs ) - -generate_enum_serilization(cpp-client-ydb_persqueue_public - ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/control_plane.h - INCLUDE_HEADERS - src/client/persqueue_public/include/control_plane.h -) - -generate_enum_serilization(cpp-client-ydb_persqueue_public - ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/read_events.h - INCLUDE_HEADERS - src/client/persqueue_public/include/read_events.h -) - -generate_enum_serilization(cpp-client-ydb_persqueue_public - ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/write_events.h - INCLUDE_HEADERS - src/client/persqueue_public/include/write_events.h -) - -generate_enum_serilization(cpp-client-ydb_persqueue_public - ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/write_session.h - INCLUDE_HEADERS - src/client/persqueue_public/include/write_session.h -) diff --git a/src/client/persqueue_public/impl/CMakeLists.txt b/src/client/persqueue_public/impl/CMakeLists.txt index 74dd435f4c..d8ab632a8b 100644 --- a/src/client/persqueue_public/impl/CMakeLists.txt +++ b/src/client/persqueue_public/impl/CMakeLists.txt @@ -12,6 +12,7 @@ target_link_libraries(client-ydb_persqueue_public-impl PUBLIC impl-ydb_internal-make_request client-ydb_common_client-impl client-ydb_driver + client-ydb_persqueue_public-include string_utils-misc ) diff --git a/src/client/persqueue_public/impl/persqueue.cpp b/src/client/persqueue_public/impl/persqueue.cpp index 4fad6d95f6..c785c31754 100644 --- a/src/client/persqueue_public/impl/persqueue.cpp +++ b/src/client/persqueue_public/impl/persqueue.cpp @@ -48,7 +48,7 @@ TCredentials::TCredentials(const Ydb::PersQueue::V1::Credentials& settings) break; } default: { - ythrow yexception() << "unsupported credentials type " << ::NPersQueue::ObfuscateString(ToString(Credentials_)); + ythrow yexception() << "unsupported credentials type " << ::NPersQueue::ObfuscateString(Credentials_.ShortDebugString()); } } } diff --git a/src/client/persqueue_public/include/CMakeLists.txt b/src/client/persqueue_public/include/CMakeLists.txt new file mode 100644 index 0000000000..670ef0a1bd --- /dev/null +++ b/src/client/persqueue_public/include/CMakeLists.txt @@ -0,0 +1,32 @@ +_ydb_sdk_add_library(client-ydb_persqueue_public-include) + +target_link_libraries(client-ydb_persqueue_public-include PUBLIC + yutil + api-grpc + api-grpc-draft + api-protos +) + +generate_enum_serilization(client-ydb_persqueue_public-include + ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/control_plane.h + INCLUDE_HEADERS + src/client/persqueue_public/include/control_plane.h +) + +generate_enum_serilization(client-ydb_persqueue_public-include + ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/read_events.h + INCLUDE_HEADERS + src/client/persqueue_public/include/read_events.h +) + +generate_enum_serilization(client-ydb_persqueue_public-include + ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/write_events.h + INCLUDE_HEADERS + src/client/persqueue_public/include/write_events.h +) + +generate_enum_serilization(client-ydb_persqueue_public-include + ${YDB_SDK_SOURCE_DIR}/src/client/persqueue_public/include/write_session.h + INCLUDE_HEADERS + src/client/persqueue_public/include/write_session.h +) diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index 28bd4a50e6..9f560ac2ed 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -994,7 +994,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( BreakConnectionAndReconnectImpl(EStatus::INTERNAL_ERROR, TStringBuilder() << "Got unexpected partition stream data message. Topic: " - << partitionData.topic() << ". Partition: " << partitionData.partition() + << partitionData.topic().ShortDebugString() << ". Partition: " << partitionData.partition() << " AssignId: " << partitionData.cookie().assign_id(), deferred); return; @@ -1031,9 +1031,9 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( if (firstOffset == std::numeric_limits::max()) { BreakConnectionAndReconnectImpl(EStatus::INTERNAL_ERROR, TStringBuilder() << "Got empty data message. Topic: " - << partitionData.topic() + << partitionData.topic().ShortDebugString() << ". Partition: " << partitionData.partition() - << " message: " << msg, + << " message: " << msg.ShortDebugString(), deferred); return; } @@ -1042,7 +1042,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( if (!CookieMapping.AddMapping(cookie)) { BreakConnectionAndReconnectImpl(EStatus::INTERNAL_ERROR, TStringBuilder() << "Got unexpected data message. Topic: " - << partitionData.topic() + << partitionData.topic().ShortDebugString() << ". Partition: " << partitionData.partition() << ". Cookie mapping already has such cookie", deferred); @@ -1150,7 +1150,7 @@ inline void TSingleClusterReadSessionImpl::OnReadDoneImpl( TDeferredActions& deferred) { Y_ABORT_UNLESS(Lock.IsLocked()); - LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Committed response: " << msg); + LOG_LAZY(Log, TLOG_DEBUG, GetLogPrefix() << "Committed response: " << msg.ShortDebugString()); std::map>> partitionStreams; for (const Ydb::PersQueue::V1::CommitCookie& cookieProto : msg.cookies()) { diff --git a/src/client/topic/ut/basic_usage_ut.cpp b/src/client/topic/ut/basic_usage_ut.cpp index cf8792f8d6..a16b0306cc 100644 --- a/src/client/topic/ut/basic_usage_ut.cpp +++ b/src/client/topic/ut/basic_usage_ut.cpp @@ -95,102 +95,6 @@ void WriteAndReadToEndWithRestarts(TReadSessionSettings readSettings, TWriteSess } Y_UNIT_TEST_SUITE(BasicUsage) { - Y_UNIT_TEST(ConnectToYDB) { - TTopicSdkTestSetup setup(TEST_CASE_NAME); - - NYdb::TDriverConfig cfg; - cfg.SetEndpoint(TStringBuilder() << "invalid:" << setup.GetServer().GrpcPort); - cfg.SetDatabase("/Invalid"); - cfg.SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); - auto driver = NYdb::TDriver(cfg); - - { - TTopicClient client(driver); - - auto writeSettings = TWriteSessionSettings() - .Path(TEST_TOPIC) - .MessageGroupId(TEST_MESSAGE_GROUP_ID) - // TODO why retries? see LOGBROKER-8490 - .RetryPolicy(IRetryPolicy::GetNoRetryPolicy()); - auto writeSession = client.CreateWriteSession(writeSettings); - - auto event = writeSession->GetEvent(true); - UNIT_ASSERT(event && std::holds_alternative(event.value())); - } - - { - auto settings = TTopicClientSettings() - .Database({"/Root"}) - .DiscoveryEndpoint("localhost:" + std::to_string(setup.GetServer().GrpcPort)); - - TTopicClient client(driver, settings); - - auto writeSettings = TWriteSessionSettings() - .Path(TEST_TOPIC) - .MessageGroupId(TEST_MESSAGE_GROUP_ID) - .RetryPolicy(IRetryPolicy::GetNoRetryPolicy()); - auto writeSession = client.CreateWriteSession(writeSettings); - - auto event = writeSession->GetEvent(true); - UNIT_ASSERT(event && !std::holds_alternative(event.value())); - } - } - - - Y_UNIT_TEST(WriteRead) { - TTopicSdkTestSetup setup(TEST_CASE_NAME); - TTopicClient client = setup.MakeClient(); - - for (size_t i = 0; i < 100; ++i) { - auto writeSettings = TWriteSessionSettings() - .Path(TEST_TOPIC) - .ProducerId(TEST_MESSAGE_GROUP_ID) - .MessageGroupId(TEST_MESSAGE_GROUP_ID); - Cerr << ">>> open write session " << i << Endl; - auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); - UNIT_ASSERT(writeSession->Write("message_using_MessageGroupId")); - Cerr << ">>> write session " << i << " message written" << Endl; - writeSession->Close(); - Cerr << ">>> write session " << i << " closed" << Endl; - } - { - auto writeSettings = TWriteSessionSettings() - .Path(TEST_TOPIC) - .ProducerId(TEST_MESSAGE_GROUP_ID) - .PartitionId(0); - Cerr << ">>> open write session 100" << Endl; - auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); - UNIT_ASSERT(writeSession->Write("message_using_PartitionId")); - Cerr << ">>> write session 100 message written" << Endl; - writeSession->Close(); - Cerr << ">>> write session 100 closed" << Endl; - } - - { - auto readSettings = TReadSessionSettings() - .ConsumerName(TEST_CONSUMER) - .AppendTopics(TEST_TOPIC); - auto readSession = client.CreateReadSession(readSettings); - - auto event = readSession->GetEvent(true); - UNIT_ASSERT(event.has_value()); - - auto& startPartitionSession = std::get(*event); - startPartitionSession.Confirm(); - - event = readSession->GetEvent(true); - UNIT_ASSERT(event.has_value()); - - auto& dataReceived = std::get(*event); - dataReceived.Commit(); - - auto& messages = dataReceived.GetMessages(); - UNIT_ASSERT(messages.size() == 101); - UNIT_ASSERT(messages[0].GetData() == "message_using_MessageGroupId"); - UNIT_ASSERT(messages[100].GetData() == "message_using_PartitionId"); - } - } - Y_UNIT_TEST(ReadWithoutConsumerWithRestarts) { TTopicSdkTestSetup setup(TEST_CASE_NAME); auto compressor = new TSyncExecutor(); @@ -219,129 +123,6 @@ Y_UNIT_TEST_SUITE(BasicUsage) { WriteAndReadToEndWithRestarts(readSettings, writeSettings, message, count, setup, decompressor); } - Y_UNIT_TEST(MaxByteSizeEqualZero) { - TTopicSdkTestSetup setup(TEST_CASE_NAME); - TTopicClient client = setup.MakeClient(); - - auto writeSettings = TWriteSessionSettings() - .Path(TEST_TOPIC) - .MessageGroupId(TEST_MESSAGE_GROUP_ID); - auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); - UNIT_ASSERT(writeSession->Write("message")); - writeSession->Close(); - - auto readSettings = TReadSessionSettings() - .ConsumerName(TEST_CONSUMER) - .AppendTopics(TEST_TOPIC); - auto readSession = client.CreateReadSession(readSettings); - - auto event = readSession->GetEvent(true); - UNIT_ASSERT(event.has_value()); - - auto& startPartitionSession = std::get(*event); - startPartitionSession.Confirm(); - - UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvent(true, 0), TContractViolation); - UNIT_CHECK_GENERATED_EXCEPTION(readSession->GetEvents(true, std::nullopt, 0), TContractViolation); - - event = readSession->GetEvent(true, 1); - UNIT_ASSERT(event.has_value()); - - auto& dataReceived = std::get(*event); - dataReceived.Commit(); - } - - Y_UNIT_TEST(WriteAndReadSomeMessagesWithSyncCompression) { - - auto setup = std::make_shared(TEST_CASE_NAME); - - NPersQueue::TWriteSessionSettings writeSettings; - writeSettings.Path(setup->GetTestTopic()).MessageGroupId(TEST_MESSAGE_GROUP_ID); - writeSettings.Codec(NPersQueue::ECodec::RAW); - IExecutor::TPtr executor = new TSyncExecutor(); - writeSettings.CompressionExecutor(executor); - - ui64 count = 100u; - TMaybe shouldCaptureData = {true}; - - auto& client = setup->GetPersQueueClient(); - auto session = client.CreateSimpleBlockingWriteSession(writeSettings); - TString messageBase = "message----"; - TVector sentMessages; - - for (auto i = 0u; i < count; i++) { - // sentMessages.emplace_back(messageBase * (i+1) + ToString(i)); - sentMessages.emplace_back(messageBase * (200 * 1024)); - auto res = session->Write(sentMessages.back()); - UNIT_ASSERT(res); - } - { - auto sessionAdapter = NPersQueue::NTests::TSimpleWriteSessionTestAdapter( - dynamic_cast(session.get())); - if (shouldCaptureData.Defined()) { - TStringBuilder msg; - msg << "Session has captured " << sessionAdapter.GetAcquiredMessagesCount() - << " messages, capturing was expected: " << *shouldCaptureData << Endl; - UNIT_ASSERT_VALUES_EQUAL_C(sessionAdapter.GetAcquiredMessagesCount() > 0, *shouldCaptureData, msg.c_str()); - } - } - session->Close(); - - std::shared_ptr ReadSession; - - // Create topic client. - NYdb::NTopic::TTopicClient topicClient(setup->GetDriver()); - - // Create read session. - TReadSessionSettings readSettings; - readSettings - .ConsumerName(setup->GetTestConsumer()) - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(setup->GetTestTopic()); - - Cerr << "Session was created" << Endl; - - NThreading::TPromise checkedPromise = NThreading::NewPromise(); - auto totalReceived = 0u; - - auto f = checkedPromise.GetFuture(); - TAtomic check = 1; - readSettings.EventHandlers_.SimpleDataHandlers( - // [checkedPromise = std::move(checkedPromise), &check, &sentMessages, &totalReceived] - [&] - (TReadSessionEvent::TDataReceivedEvent& ev) mutable { - Y_VERIFY_S(AtomicGet(check) != 0, "check is false"); - auto& messages = ev.GetMessages(); - for (size_t i = 0u; i < messages.size(); ++i) { - auto& message = messages[i]; - UNIT_ASSERT_VALUES_EQUAL(message.GetData(), sentMessages[totalReceived]); - totalReceived++; - } - if (totalReceived == sentMessages.size()) - checkedPromise.SetValue(); - }); - - ReadSession = topicClient.CreateReadSession(readSettings); - - f.GetValueSync(); - ReadSession->Close(TDuration::MilliSeconds(10)); - AtomicSet(check, 0); - - auto status = topicClient.CommitOffset(setup->GetTestTopic(), 0, setup->GetTestConsumer(), 50); - UNIT_ASSERT(status.GetValueSync().IsSuccess()); - - auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); - auto result = topicClient.DescribeConsumer(setup->GetTestTopicPath(), setup->GetTestConsumer(), describeConsumerSettings).GetValueSync(); - UNIT_ASSERT(result.IsSuccess()); - - auto description = result.GetConsumerDescription(); - UNIT_ASSERT(description.GetPartitions().size() == 1); - auto stats = description.GetPartitions().front().GetPartitionConsumerStats(); - UNIT_ASSERT(stats.has_value()); - UNIT_ASSERT(stats->GetCommittedOffset() == 50); - } - - Y_UNIT_TEST(ReadWithRestarts) { TTopicSdkTestSetup setup(TEST_CASE_NAME); auto compressor = new TSyncExecutor(); @@ -367,358 +148,6 @@ Y_UNIT_TEST_SUITE(BasicUsage) { WriteAndReadToEndWithRestarts(readSettings, writeSettings, message, count, setup, decompressor); } - Y_UNIT_TEST(SessionNotDestroyedWhileCompressionInFlight) { - TTopicSdkTestSetup setup(TEST_CASE_NAME); - - // controlled executor - auto stepByStepExecutor = CreateThreadPoolManagedExecutor(1); - - // Create topic client. - TTopicClient topicClient = setup.MakeClient(); - - NThreading::TPromise promiseToWrite = NThreading::NewPromise(); - auto futureWrite = promiseToWrite.GetFuture(); - - NThreading::TPromise promiseToRead = NThreading::NewPromise(); - auto futureRead = promiseToRead.GetFuture(); - - TWriteSessionSettings writeSettings; - writeSettings.Path(TEST_TOPIC) - .MessageGroupId(TEST_MESSAGE_GROUP_ID) - .ProducerId(TEST_MESSAGE_GROUP_ID) - .CompressionExecutor(stepByStepExecutor); - - // Create read session. - TReadSessionSettings readSettings; - readSettings - .ConsumerName(TEST_CONSUMER) - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(TEST_TOPIC) - .DecompressionExecutor(stepByStepExecutor); - - auto f = std::async(std::launch::async, - [readSettings, writeSettings, &topicClient, - promiseToWrite = std::move(promiseToWrite), - promiseToRead = std::move(promiseToRead)]() mutable { - { - auto writeSession = topicClient.CreateSimpleBlockingWriteSession(writeSettings); - std::string message(2'000, 'x'); - bool res = writeSession->Write(message); - UNIT_ASSERT(res); - writeSession->Close(TDuration::Seconds(10)); - } - promiseToWrite.SetValue(); - Cerr << ">>>TEST: write promise set " << Endl; - - { - NThreading::TPromise promise = NThreading::NewPromise(); - auto future = promise.GetFuture(); - - readSettings.EventHandlers_.SimpleDataHandlers( - [promise = std::move(promise)](TReadSessionEvent::TDataReceivedEvent& ev) mutable { - ev.Commit(); - promise.SetValue(); - Cerr << ">>>TEST: get read event " << Endl; - }); - - auto readSession = topicClient.CreateReadSession(readSettings); - future.Wait(); - readSession->Close(TDuration::Seconds(10)); - } - promiseToRead.SetValue(); - Cerr << ">>>TEST: read promise set " << Endl; - }); - - - // - // auxiliary functions for decompressor and handler control - // - auto WaitTasks = [&](auto f, size_t c) { - while (f() < c) { - Sleep(TDuration::MilliSeconds(100)); - }; - }; - auto WaitPlannedTasks = [&](auto e, size_t count) { - WaitTasks([&]() { return e->GetPlannedCount(); }, count); - }; - auto WaitExecutedTasks = [&](auto e, size_t count) { - WaitTasks([&]() { return e->GetExecutedCount(); }, count); - }; - - auto RunTasks = [&](auto e, const std::vector& tasks) { - size_t n = tasks.size(); - Cerr << ">>>TEST in RunTasks: before WaitPlannedTasks" << Endl; - WaitPlannedTasks(e, n); - Cerr << ">>>TEST in RunTasks: before WaitExecutedTasks" << Endl; - size_t completed = e->GetExecutedCount(); - e->StartFuncs(tasks); - WaitExecutedTasks(e, completed + n); - }; - - UNIT_ASSERT(!futureWrite.HasValue()); - Cerr << ">>>TEST: future write has no value " << Endl; - RunTasks(stepByStepExecutor, {0}); // Run compression task. - RunTasks(stepByStepExecutor, {1}); // Run send task. - futureWrite.GetValueSync(); - UNIT_ASSERT(futureWrite.HasValue()); - Cerr << ">>>TEST: future write has value " << Endl; - - UNIT_ASSERT(!futureRead.HasValue()); - Cerr << ">>>TEST: future read has no value " << Endl; - RunTasks(stepByStepExecutor, {2}); // Run decompression task. - futureRead.GetValueSync(); - UNIT_ASSERT(futureRead.HasValue()); - Cerr << ">>>TEST: future read has value " << Endl; - - f.get(); - - Cerr << ">>> TEST: gracefully closed" << Endl; - } - - Y_UNIT_TEST(SessionNotDestroyedWhileUserEventHandlingInFlight) { - TTopicSdkTestSetup setup(TEST_CASE_NAME); - - // controlled executor - auto stepByStepExecutor = CreateThreadPoolManagedExecutor(1); - - // Create topic client. - TTopicClient topicClient = setup.MakeClient(); - - // NThreading::TPromise promiseToWrite = NThreading::NewPromise(); - // auto futureWrite = promiseToWrite.GetFuture(); - - NThreading::TPromise promiseToRead = NThreading::NewPromise(); - auto futureRead = promiseToRead.GetFuture(); - - auto writeSettings = TWriteSessionSettings() - .Path(TEST_TOPIC) - .MessageGroupId(TEST_MESSAGE_GROUP_ID) - .ProducerId(TEST_MESSAGE_GROUP_ID); - - auto writeSession = topicClient.CreateSimpleBlockingWriteSession(writeSettings); - std::string message(2'000, 'x'); - bool res = writeSession->Write(message); - UNIT_ASSERT(res); - writeSession->Close(TDuration::Seconds(10)); - - // writeSettings.EventHandlers_ - // .HandlersExecutor(stepByStepExecutor); - - // Create read session. - auto readSettings = TReadSessionSettings() - .ConsumerName(TEST_CONSUMER) - .MaxMemoryUsageBytes(1_MB) - .AppendTopics(TEST_TOPIC); - - readSettings.EventHandlers_ - .HandlersExecutor(stepByStepExecutor); - - auto f = std::async(std::launch::async, - [readSettings, /*writeSettings,*/ &topicClient, - // promiseToWrite = std::move(promiseToWrite), - promiseToRead = std::move(promiseToRead)]() mutable { - // { - // std::shared_ptr token; - // writeSettings.EventHandlers_.CommonHandler([token](TWriteSessionEvent::TEvent& event){ - // Cerr << ">>>TEST: in CommonHandler " << Endl; - - // if (std::holds_alternative(event)) { - // *token = std::move(std::get(event).ContinuationToken); - // } - // }); - - // auto writeSession = topicClient.CreateWriteSession(writeSettings); - // std::string message(2'000, 'x'); - // writeSession->WaitEvent().Wait(); - // writeSession->Write(std::move(*token), message); - // writeSession->WaitEvent().Wait(); - // writeSession->Close(TDuration::Seconds(10)); - // } - // promiseToWrite.SetValue(); - // Cerr << ">>>TEST: write promise set " << Endl; - - { - NThreading::TPromise promise = NThreading::NewPromise(); - auto future = promise.GetFuture(); - - readSettings.EventHandlers_.SimpleDataHandlers( - [promise = std::move(promise)](TReadSessionEvent::TDataReceivedEvent& ev) mutable { - Cerr << ">>>TEST: in SimpleDataHandlers " << Endl; - ev.Commit(); - promise.SetValue(); - }); - - auto readSession = topicClient.CreateReadSession(readSettings); - future.Wait(); - readSession->Close(TDuration::Seconds(10)); - } - promiseToRead.SetValue(); - Cerr << ">>>TEST: read promise set " << Endl; - }); - - - // - // auxiliary functions for decompressor and handler control - // - auto WaitTasks = [&](auto f, size_t c) { - while (f() < c) { - Sleep(TDuration::MilliSeconds(100)); - }; - }; - auto WaitPlannedTasks = [&](auto e, size_t count) { - WaitTasks([&]() { return e->GetPlannedCount(); }, count); - }; - auto WaitExecutedTasks = [&](auto e, size_t count) { - WaitTasks([&]() { return e->GetExecutedCount(); }, count); - }; - - auto RunTasks = [&](auto e, const std::vector& tasks) { - size_t n = tasks.size(); - Cerr << ">>>TEST in RunTasks: before WaitPlannedTasks" << Endl; - WaitPlannedTasks(e, n); - Cerr << ">>>TEST in RunTasks: before WaitExecutedTasks" << Endl; - size_t completed = e->GetExecutedCount(); - e->StartFuncs(tasks); - WaitExecutedTasks(e, completed + n); - }; - - // RunTasks(stepByStepExecutor, {0}); - // UNIT_ASSERT(!futureWrite.HasValue()); - // Cerr << ">>>TEST: future write has no value " << Endl; - // RunTasks(stepByStepExecutor, {1}); - // futureWrite.GetValueSync(); - // UNIT_ASSERT(futureWrite.HasValue()); - // Cerr << ">>>TEST: future write has value " << Endl; - - UNIT_ASSERT(!futureRead.HasValue()); - Cerr << ">>>TEST: future read has no value " << Endl; - // 0: TStartPartitionSessionEvent - RunTasks(stepByStepExecutor, {0}); - // 1: TDataReceivedEvent - RunTasks(stepByStepExecutor, {1}); - futureRead.GetValueSync(); - UNIT_ASSERT(futureRead.HasValue()); - Cerr << ">>>TEST: future read has value " << Endl; - - f.get(); - - Cerr << ">>> TEST: gracefully closed" << Endl; - } - - Y_UNIT_TEST(ReadSessionCorrectClose) { - - auto setup = std::make_shared(TEST_CASE_NAME); - - NPersQueue::TWriteSessionSettings writeSettings; - writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); - writeSettings.Codec(NPersQueue::ECodec::RAW); - IExecutor::TPtr executor = new TSyncExecutor(); - writeSettings.CompressionExecutor(executor); - - auto& client = setup->GetPersQueueClient(); - auto session = client.CreateSimpleBlockingWriteSession(writeSettings); - - ui32 count = 7000; - std::string message(2'000, 'x'); - for (ui32 i = 1; i <= count; ++i) { - bool res = session->Write(message); - UNIT_ASSERT(res); - } - bool res = session->Close(TDuration::Seconds(30)); - UNIT_ASSERT(res); - - std::shared_ptr ReadSession; - - // Create topic client. - NYdb::NTopic::TTopicClient topicClient(setup->GetDriver()); - - // Create read session. - NYdb::NTopic::TReadSessionSettings readSettings; - readSettings - .ConsumerName(setup->GetTestConsumer()) - .MaxMemoryUsageBytes(1_MB) - .Decompress(false) - .RetryPolicy(NYdb::NTopic::IRetryPolicy::GetNoRetryPolicy()) - .AppendTopics(setup->GetTestTopic()); - - readSettings.EventHandlers_.SimpleDataHandlers( - [] - (NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent& ev) mutable { - Cerr << ">>> Got TDataReceivedEvent" << Endl; - ev.Commit(); - }); - - Cerr << ">>> TEST: Create session" << Endl; - - ReadSession = topicClient.CreateReadSession(readSettings); - - Sleep(TDuration::MilliSeconds(50)); - - ReadSession->Close(); - ReadSession = nullptr; - Cerr << ">>> TEST: Session gracefully closed" << Endl; - - Sleep(TDuration::Seconds(5)); - } - - Y_UNIT_TEST(ConfirmPartitionSessionWithCommitOffset) { - // TStartPartitionSessionEvent::Confirm(readOffset, commitOffset) should work, - // if commitOffset passed to Confirm is greater than the offset committed previously by the consumer. - // https://st.yandex-team.ru/KIKIMR-23015 - - auto setup = TTopicSdkTestSetup(TEST_CASE_NAME); - - { - // Write 2 messages: - auto settings = NTopic::TWriteSessionSettings() - .Path(setup.GetTopicPath()) - .MessageGroupId(TEST_MESSAGE_GROUP_ID) - .ProducerId(TEST_MESSAGE_GROUP_ID); - auto client = setup.MakeClient(); - auto writer = client.CreateSimpleBlockingWriteSession(settings); - writer->Write("message"); - writer->Write("message"); - writer->Close(); - } - - { - // Read messages: - auto settings = NTopic::TReadSessionSettings() - .ConsumerName(TEST_CONSUMER) - .AppendTopics(std::string(setup.GetTopicPath())); - - auto client = setup.MakeClient(); - auto reader = client.CreateReadSession(settings); - - { - // Start partition session and request to read from offset 1 and commit offset 1: - auto event = reader->GetEvent(true); - UNIT_ASSERT(event.has_value()); - UNIT_ASSERT(std::holds_alternative(*event)); - auto& startPartitionSession = std::get(*event); - startPartitionSession.Confirm(/*readOffset=*/ 1, /*commitOffset=*/ 1); - } - - { - // Receive a message with offset 1 and commit it: - auto event = reader->GetEvent(true); - UNIT_ASSERT(event.has_value()); - UNIT_ASSERT(std::holds_alternative(*event)); - auto& dataReceived = std::get(*event); - - // Here we should commit range [1, 2), not [0, 2): - dataReceived.Commit(); - } - - { - // And then get a TCommitOffsetAcknowledgementEvent: - auto event = reader->GetEvent(true); - UNIT_ASSERT(event.has_value()); - UNIT_ASSERT(std::holds_alternative(*event)); - } - } - } - Y_UNIT_TEST(ConflictingWrites) { TTopicSdkTestSetup setup(TEST_CASE_NAME); @@ -757,261 +186,6 @@ Y_UNIT_TEST_SUITE(BasicUsage) { UNIT_ASSERT_VALUES_EQUAL(stats->GetEndOffset(), count); } - - Y_UNIT_TEST(TWriteSession_WriteEncoded) { - // This test was adapted from ydb_persqueue tests. - // It writes 4 messages: 2 with default codec, 1 with explicitly set GZIP codec, 1 with RAW codec. - // The last message MUST be sent in a separate WriteRequest, as it has a codec field applied for all messages in the request. - // This separation currently happens in TWriteSessionImpl::SendImpl method. - - auto setup = std::make_shared(TEST_CASE_NAME); - auto client = setup->MakeClient(); - auto settings = TWriteSessionSettings() - .Path(TEST_TOPIC) - .MessageGroupId(TEST_MESSAGE_GROUP_ID); - - size_t batchSize = 100000000; - settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. - settings.BatchFlushSizeBytes(batchSize); - auto writer = client.CreateWriteSession(settings); - TString message = "message"; - TString packed; - { - TStringOutput so(packed); - TZLibCompress oss(&so, ZLib::GZip, 6); - oss << message; - } - - Cerr << message << " " << packed << "\n"; - - { - auto event = *writer->GetEvent(true); - UNIT_ASSERT(!writer->WaitEvent().Wait(TDuration::Seconds(1))); - auto ev = writer->WaitEvent(); - UNIT_ASSERT(std::holds_alternative(event)); - auto continueToken = std::move(std::get(event).ContinuationToken); - writer->Write(std::move(continueToken), message); - UNIT_ASSERT(ev.Wait(TDuration::Seconds(1))); - } - { - auto event = *writer->GetEvent(true); - UNIT_ASSERT(std::holds_alternative(event)); - auto continueToken = std::move(std::get(event).ContinuationToken); - writer->Write(std::move(continueToken), ""); - } - { - auto event = *writer->GetEvent(true); - UNIT_ASSERT(std::holds_alternative(event)); - auto continueToken = std::move(std::get(event).ContinuationToken); - writer->WriteEncoded(std::move(continueToken), packed, ECodec::GZIP, message.size()); - } - - ui32 acks = 0, tokens = 0; - while(acks < 4 || tokens < 2) { - auto event = *writer->GetEvent(true); - if (std::holds_alternative(event)) { - acks += std::get(event).Acks.size(); - } - if (std::holds_alternative(event)) { - if (tokens == 0) { - auto continueToken = std::move(std::get(event).ContinuationToken); - writer->WriteEncoded(std::move(continueToken), "", ECodec::RAW, 0); - } - ++tokens; - } - Cerr << "GOT EVENT " << acks << " " << tokens << "\n"; - } - UNIT_ASSERT(!writer->WaitEvent().Wait(TDuration::Seconds(5))); - - UNIT_ASSERT_VALUES_EQUAL(acks, 4); - UNIT_ASSERT_VALUES_EQUAL(tokens, 2); - - auto readSettings = TReadSessionSettings() - .ConsumerName(TEST_CONSUMER) - .AppendTopics(TEST_TOPIC); - std::shared_ptr readSession = client.CreateReadSession(readSettings); - ui32 readMessageCount = 0; - while (readMessageCount < 4) { - Cerr << "Get event on client\n"; - auto event = *readSession->GetEvent(true); - std::visit(TOverloaded { - [&](TReadSessionEvent::TDataReceivedEvent& event) { - for (auto& message: event.GetMessages()) { - std::string sourceId = message.GetMessageGroupId(); - ui32 seqNo = message.GetSeqNo(); - UNIT_ASSERT_VALUES_EQUAL(readMessageCount + 1, seqNo); - ++readMessageCount; - UNIT_ASSERT_VALUES_EQUAL(message.GetData(), (seqNo % 2) == 1 ? "message" : ""); - } - }, - [&](TReadSessionEvent::TCommitOffsetAcknowledgementEvent&) { - UNIT_FAIL("no commits in test"); - }, - [&](TReadSessionEvent::TStartPartitionSessionEvent& event) { - event.Confirm(); - }, - [&](TReadSessionEvent::TStopPartitionSessionEvent& event) { - event.Confirm(); - }, - [&](TReadSessionEvent::TEndPartitionSessionEvent& event) { - event.Confirm(); - }, - [&](TReadSessionEvent::TPartitionSessionStatusEvent&) { - UNIT_FAIL("Test does not support lock sessions yet"); - }, - [&](TReadSessionEvent::TPartitionSessionClosedEvent&) { - UNIT_FAIL("Test does not support lock sessions yet"); - }, - [&](TSessionClosedEvent&) { - UNIT_FAIL("Session closed"); - } - - }, event); - } - } } // Y_UNIT_TEST_SUITE(BasicUsage) -Y_UNIT_TEST_SUITE(TSettingsValidation) { - enum class EExpectedTestResult { - SUCCESS, - FAIL_ON_SDK, - FAIL_ON_RPC - }; - - Y_UNIT_TEST(TestDifferentDedupParams) { - TTopicSdkTestSetup setup(TEST_CASE_NAME); - setup.GetServer().EnableLogs({ - NKikimrServices::PERSQUEUE, NKikimrServices::PERSQUEUE_READ_BALANCER, NKikimrServices::PQ_WRITE_PROXY, NKikimrServices::PQ_PARTITION_CHOOSER}, - NActors::NLog::PRI_ERROR); - - - auto client = setup.MakeClient(); - ui64 producerIndex = 0u; - auto runTest = [&](TString producer, TString msgGroup, const std::optional& useDedup, bool useSeqNo, EExpectedTestResult result) ->bool - { - TWriteSessionSettings writeSettings; - writeSettings.Path(setup.GetTopicPath()).Codec(NTopic::ECodec::RAW); - TString useDedupStr = useDedup.has_value() ? ToString(*useDedup) : ""; - if (producer) { - producer += ToString(producerIndex); - } - if (!msgGroup.empty()) { - msgGroup += ToString(producerIndex); - } - writeSettings.ProducerId(producer).MessageGroupId(msgGroup); - producerIndex++; - Cerr.Flush(); - Sleep(TDuration::MilliSeconds(250)); - Cerr << "=== === START TEST. Producer = '" << producer << "', MsgGroup = '" << msgGroup << "', useDedup: " - << useDedupStr << ", manual SeqNo: " << useSeqNo << Endl; - - try { - if (useDedup.has_value()) { - writeSettings.DeduplicationEnabled(useDedup); - } - auto session = client.CreateWriteSession(writeSettings); - TMaybe token; - ui64 seqNo = 1u; - ui64 written = 0; - while (written < 10) { - auto event = session->GetEvent(true); - if (std::holds_alternative(event.value())) { - auto closed = std::get(*event); - Cerr << "Session failed with error: " << closed.DebugString() << Endl; - UNIT_ASSERT(result == EExpectedTestResult::FAIL_ON_RPC); - return false; - } else if (std::holds_alternative(event.value())) { - token = std::move(std::get(*event).ContinuationToken); - if (useSeqNo) { - session->Write(std::move(*token), "data", seqNo++); - } else { - session->Write(std::move(*token), "data"); - } - continue; - } else { - UNIT_ASSERT(std::holds_alternative(*event)); - const auto& acks = std::get(*event); - for (const auto& ack : acks.Acks) { - UNIT_ASSERT(ack.State == TWriteSessionEvent::TWriteAck::EES_WRITTEN); - } - written += acks.Acks.size(); - } - } - } catch(const NYdb::TContractViolation& ex) { - Cerr << "Test fails on contract validation: " << ex.what() << Endl; - UNIT_ASSERT(result == EExpectedTestResult::FAIL_ON_SDK); - return false; - } - Cerr << "=== === END TEST (supposed ok)=== ===\n\n"; - UNIT_ASSERT(result == EExpectedTestResult::SUCCESS); - return true; - }; - // Normal scenarios: - // Most common: - TVector producers = {"producer", ""}; - TVector> messageGroup = {Nothing(), "producer", "messageGroup", ""}; - TVector> useDedupVariants = {Nothing(), true, false}; - TVector manSeqNoVariants = {true, false}; - runTest("producer", {}, {}, false, EExpectedTestResult::SUCCESS); - runTest("producer", {}, {}, true, EExpectedTestResult::SUCCESS); - // Enable dedup (doesnt take affect anything as it is enabled anyway) - runTest("producer", {}, true, true, EExpectedTestResult::SUCCESS); - runTest("producer", {}, true, false, EExpectedTestResult::SUCCESS); - - //No producer, do dedup - runTest({}, {}, {}, false, EExpectedTestResult::SUCCESS); - // manual seqNo with no-dedup - error - runTest({}, {}, {}, true, EExpectedTestResult::FAIL_ON_SDK); - // No producer but do enable dedup - runTest({}, {}, true, true, EExpectedTestResult::SUCCESS); - runTest({}, {}, true, false, EExpectedTestResult::SUCCESS); - - // MsgGroup = producer with explicit dedup enabling or not - runTest("producer", "producer", {}, false, EExpectedTestResult::SUCCESS); - runTest("producer", "producer", {}, true, EExpectedTestResult::SUCCESS); - runTest("producer", "producer", true, true, EExpectedTestResult::SUCCESS); - runTest("producer", "producer", true, false, EExpectedTestResult::SUCCESS); - - //Bad scenarios - // MsgGroup != producer, triggers error - runTest("producer", "msgGroup", {}, false, EExpectedTestResult::FAIL_ON_SDK); - runTest("producer", "msgGroup", {}, true, EExpectedTestResult::FAIL_ON_SDK); - runTest("producer", "msgGroup", true, true, EExpectedTestResult::FAIL_ON_SDK); - runTest("producer", "msgGroup", true, false, EExpectedTestResult::FAIL_ON_SDK); - - //Set producer or msgGroupId but disnable dedup: - runTest("producer", {}, false, true, EExpectedTestResult::FAIL_ON_SDK); - runTest("producer", {}, false, false, EExpectedTestResult::FAIL_ON_SDK); - runTest({}, "msgGroup", false, true, EExpectedTestResult::FAIL_ON_SDK); - runTest({}, "msgGroup", false, false, EExpectedTestResult::FAIL_ON_SDK); - - //Use msgGroupId as producerId, enable dedup - runTest({}, "msgGroup", true, true, EExpectedTestResult::SUCCESS); - runTest({}, "msgGroup", true, false, EExpectedTestResult::SUCCESS); - - - //Specify msg groupId and don't specify deduplication. Should work with dedup enable - runTest({}, "msgGroup", {}, true, EExpectedTestResult::SUCCESS); - runTest({}, "msgGroup", {}, false, EExpectedTestResult::SUCCESS); - } - - Y_UNIT_TEST(ValidateSettingsFailOnStart) { - TTopicSdkTestSetup setup(TEST_CASE_NAME); - TTopicClient client = setup.MakeClient(); - - auto readSettings = TReadSessionSettings() - .ConsumerName(TEST_CONSUMER) - .MaxMemoryUsageBytes(0) - .AppendTopics(TEST_TOPIC); - - auto readSession = client.CreateReadSession(readSettings); - auto event = readSession->GetEvent(true); - UNIT_ASSERT(event.has_value()); - - auto& closeEvent = std::get(*event); - UNIT_ASSERT(closeEvent.DebugString().contains("Too small max memory usage")); - } - -} // Y_UNIT_TEST_SUITE(TSettingsValidation) - } // namespace diff --git a/tests/integration/CMakeLists.txt b/tests/integration/CMakeLists.txt index 78d656ade6..594bd829de 100644 --- a/tests/integration/CMakeLists.txt +++ b/tests/integration/CMakeLists.txt @@ -3,3 +3,4 @@ add_subdirectory(bulk_upsert) add_subdirectory(server_restart) add_subdirectory(sessions) add_subdirectory(sessions_pool) +add_subdirectory(topic) diff --git a/tests/integration/topic/CMakeLists.txt b/tests/integration/topic/CMakeLists.txt new file mode 100644 index 0000000000..62d372aaa7 --- /dev/null +++ b/tests/integration/topic/CMakeLists.txt @@ -0,0 +1,11 @@ +add_ydb_test(NAME topic_it GTEST + SOURCES + basic_usage.cpp + LINK_LIBRARIES + yutil + YDB-CPP-SDK::Topic + cpp-client-ydb_persqueue_public + api-grpc + LABELS + integration +) diff --git a/tests/integration/topic/basic_usage.cpp b/tests/integration/topic/basic_usage.cpp new file mode 100644 index 0000000000..52ec0dfe72 --- /dev/null +++ b/tests/integration/topic/basic_usage.cpp @@ -0,0 +1,1063 @@ +#include + +#include + +#include +#include + +#include +#include + +#include + +#include + +namespace NYdb::NPersQueue::NTests { + +class TSimpleWriteSessionTestAdapter { +public: + TSimpleWriteSessionTestAdapter(NPersQueue::TSimpleBlockingWriteSession* session); + std::uint64_t GetAcquiredMessagesCount() const; + +private: + NPersQueue::TSimpleBlockingWriteSession* Session; +}; + +TSimpleWriteSessionTestAdapter::TSimpleWriteSessionTestAdapter(NPersQueue::TSimpleBlockingWriteSession* session) + : Session(session) +{} + +std::uint64_t TSimpleWriteSessionTestAdapter::GetAcquiredMessagesCount() const { + if (Session->Writer) + return Session->Writer->TryGetImpl()->MessagesAcquired; + else + return 0; +} + +} + +namespace NYdb::NTopic::NTests { + +class TManagedExecutor : public IExecutor { +public: + using TExecutorPtr = IExecutor::TPtr; + + explicit TManagedExecutor(TExecutorPtr executor); + + bool IsAsync() const override; + void Post(TFunction&& f) override; + + void StartFuncs(const std::vector& indicies); + + size_t GetFuncsCount() const; + + size_t GetPlannedCount() const; + size_t GetRunningCount() const; + size_t GetExecutedCount() const; + + void RunAllTasks(); + +private: + void DoStart() override; + + TFunction MakeTask(TFunction func); + void RunTask(TFunction&& func); + + TExecutorPtr Executor; + mutable std::mutex Mutex; + std::vector Funcs; + std::atomic Planned = 0; + std::atomic Running = 0; + std::atomic Executed = 0; +}; + +TManagedExecutor::TManagedExecutor(TExecutorPtr executor) : + Executor{std::move(executor)} +{ +} + +bool TManagedExecutor::IsAsync() const +{ + return Executor->IsAsync(); +} + +void TManagedExecutor::Post(TFunction &&f) +{ + std::lock_guard lock(Mutex); + + Funcs.push_back(std::move(f)); + ++Planned; +} + +void TManagedExecutor::DoStart() +{ + Executor->Start(); +} + +auto TManagedExecutor::MakeTask(TFunction func) -> TFunction +{ + return [this, func = std::move(func)]() { + ++Running; + + func(); + + --Running; + ++Executed; + }; +} + +void TManagedExecutor::RunTask(TFunction&& func) +{ + Y_ABORT_UNLESS(Planned > 0); + --Planned; + Executor->Post(MakeTask(std::move(func))); +} + +void TManagedExecutor::StartFuncs(const std::vector& indicies) +{ + std::lock_guard lock(Mutex); + + for (auto index : indicies) { + Y_ABORT_UNLESS(index < Funcs.size()); + Y_ABORT_UNLESS(Funcs[index]); + + RunTask(std::move(Funcs[index])); + } +} + +size_t TManagedExecutor::GetFuncsCount() const +{ + std::lock_guard lock(Mutex); + + return Funcs.size(); +} + +size_t TManagedExecutor::GetPlannedCount() const +{ + return Planned; +} + +size_t TManagedExecutor::GetRunningCount() const +{ + return Running; +} + +size_t TManagedExecutor::GetExecutedCount() const +{ + return Executed; +} + +void TManagedExecutor::RunAllTasks() +{ + std::lock_guard lock(Mutex); + + for (auto& func : Funcs) { + if (func) { + RunTask(std::move(func)); + } + } +} + +TIntrusivePtr CreateThreadPoolManagedExecutor(size_t threads) +{ + return MakeIntrusive(NYdb::NTopic::CreateThreadPoolExecutor(threads)); +} + +TIntrusivePtr CreateSyncManagedExecutor() +{ + return MakeIntrusive(NYdb::NTopic::CreateSyncExecutor()); +} + +class TTopicTestFixture : public ::testing::Test { +protected: + void SetUp() override { + TTopicClient client(MakeDriver()); + client.DropTopic(GetTopicPath()).GetValueSync(); + + CreateTopic(GetTopicPath()); + } + + void TearDown() override { + DropTopic(GetTopicPath()); + } + + void CreateTopic(const std::string& path, const std::string& consumer = "test-consumer", size_t partitionCount = 1, + std::optional maxPartitionCount = std::nullopt) { + TTopicClient client(MakeDriver()); + + TCreateTopicSettings topics; + topics + .BeginConfigurePartitioningSettings() + .MinActivePartitions(partitionCount) + .MaxActivePartitions(maxPartitionCount.value_or(partitionCount)); + + if (maxPartitionCount.has_value() && maxPartitionCount.value() > partitionCount) { + topics + .BeginConfigurePartitioningSettings() + .BeginConfigureAutoPartitioningSettings() + .Strategy(EAutoPartitioningStrategy::ScaleUp); + } + + TConsumerSettings consumers(topics, consumer); + topics.AppendConsumers(consumers); + + auto status = client.CreateTopic(path, topics).GetValueSync(); + ASSERT_TRUE(status.IsSuccess()); + } + + std::string GetTopicPath() { + const testing::TestInfo* const testInfo = testing::UnitTest::GetInstance()->current_test_info(); + + return std::string(testInfo->test_suite_name()) + "/" + std::string(testInfo->name()) + "/test-topic"; + } + + void DropTopic(const std::string& path) { + TTopicClient client(MakeDriver()); + auto status = client.DropTopic(path).GetValueSync(); + ASSERT_TRUE(status.IsSuccess()); + } + + TDriver MakeDriver() { + auto cfg = NYdb::TDriverConfig() + .SetEndpoint(std::getenv("YDB_ENDPOINT")) + .SetDatabase(std::getenv("YDB_DATABASE")) + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); + + return NYdb::TDriver(cfg); + } +}; + +class BasicUsage : public TTopicTestFixture {}; + +TEST_F(BasicUsage, ConnectToYDB) { + auto cfg = NYdb::TDriverConfig() + .SetEndpoint("invalid:2136") + .SetDatabase("/Invalid") + .SetLog(std::unique_ptr(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG).Release())); + auto driver = NYdb::TDriver(cfg); + + { + TTopicClient client(driver); + + auto writeSettings = TWriteSessionSettings() + .Path(GetTopicPath()) + .MessageGroupId("test-message_group_id") + // TODO why retries? see LOGBROKER-8490 + .RetryPolicy(IRetryPolicy::GetNoRetryPolicy()); + auto writeSession = client.CreateWriteSession(writeSettings); + + auto event = writeSession->GetEvent(true); + ASSERT_TRUE(event && std::holds_alternative(event.value())); + } + + { + auto settings = TTopicClientSettings() + .Database(std::getenv("YDB_DATABASE")) + .DiscoveryEndpoint(std::getenv("YDB_ENDPOINT")); + + TTopicClient client(driver, settings); + + auto writeSettings = TWriteSessionSettings() + .Path(GetTopicPath()) + .MessageGroupId("test-message_group_id") + .RetryPolicy(IRetryPolicy::GetNoRetryPolicy()); + auto writeSession = client.CreateWriteSession(writeSettings); + + auto event = writeSession->GetEvent(true); + ASSERT_TRUE(event && !std::holds_alternative(event.value())); + } +} + +TEST_F(BasicUsage, WriteRead) { + auto driver = MakeDriver(); + + TTopicClient client(driver); + + for (size_t i = 0; i < 100; ++i) { + auto writeSettings = TWriteSessionSettings() + .Path(GetTopicPath()) + .ProducerId("test-message_group_id") + .MessageGroupId("test-message_group_id"); + std::cerr << ">>> open write session " << i << std::endl; + auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); + ASSERT_TRUE(writeSession->Write("message_using_MessageGroupId")); + std::cerr << ">>> write session " << i << " message written" << std::endl; + writeSession->Close(); + std::cerr << ">>> write session " << i << " closed" << std::endl; + } + { + auto writeSettings = TWriteSessionSettings() + .Path(GetTopicPath()) + .ProducerId("test-message_group_id") + .PartitionId(0); + std::cerr << ">>> open write session 100" << std::endl; + auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); + ASSERT_TRUE(writeSession->Write("message_using_PartitionId")); + std::cerr << ">>> write session 100 message written" << std::endl; + writeSession->Close(); + std::cerr << ">>> write session 100 closed" << std::endl; + } + + { + auto readSettings = TReadSessionSettings() + .ConsumerName("test-consumer") + .AppendTopics(GetTopicPath()); + auto readSession = client.CreateReadSession(readSettings); + + auto event = readSession->GetEvent(true); + ASSERT_TRUE(event.has_value()); + + auto& startPartitionSession = std::get(*event); + startPartitionSession.Confirm(); + + event = readSession->GetEvent(true); + ASSERT_TRUE(event.has_value()); + + auto& dataReceived = std::get(*event); + dataReceived.Commit(); + + auto& messages = dataReceived.GetMessages(); + ASSERT_EQ(messages.size(), 101u); + ASSERT_EQ(messages[0].GetData(), "message_using_MessageGroupId"); + ASSERT_EQ(messages[100].GetData(), "message_using_PartitionId"); + } +} + +TEST_F(BasicUsage, MaxByteSizeEqualZero) { + auto driver = MakeDriver(); + + TTopicClient client(driver); + + auto writeSettings = TWriteSessionSettings() + .Path(GetTopicPath()) + .MessageGroupId("test-message_group_id"); + auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); + ASSERT_TRUE(writeSession->Write("message")); + writeSession->Close(); + + auto readSettings = TReadSessionSettings() + .ConsumerName("test-consumer") + .AppendTopics(GetTopicPath()); + auto readSession = client.CreateReadSession(readSettings); + + auto event = readSession->GetEvent(true); + ASSERT_TRUE(event.has_value()); + + auto& startPartitionSession = std::get(*event); + startPartitionSession.Confirm(); + + ASSERT_THROW(readSession->GetEvent(true, 0), TContractViolation); + ASSERT_THROW(readSession->GetEvents(true, std::nullopt, 0), TContractViolation); + + event = readSession->GetEvent(true, 1); + ASSERT_TRUE(event.has_value()); + + auto& dataReceived = std::get(*event); + dataReceived.Commit(); +} + +TEST_F(BasicUsage, WriteAndReadSomeMessagesWithSyncCompression) { + auto driver = MakeDriver(); + + IExecutor::TPtr executor = new TSyncExecutor(); + auto writeSettings = NPersQueue::TWriteSessionSettings() + .Path(GetTopicPath()) + .MessageGroupId("test-message_group_id") + .Codec(NPersQueue::ECodec::RAW) + .CompressionExecutor(executor); + + std::uint64_t count = 100u; + std::optional shouldCaptureData = {true}; + + NPersQueue::TPersQueueClient client(driver); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + std::string messageBase = "message----"; + std::vector sentMessages; + + for (auto i = 0u; i < count; i++) { + // sentMessages.emplace_back(messageBase * (i+1) + ToString(i)); + std::ostringstream os; + for(int i = 0; i < 200 * 1024; i++) { + os << messageBase; + } + sentMessages.emplace_back(os.str()); + auto res = session->Write(sentMessages.back()); + ASSERT_TRUE(res); + } + { + auto sessionAdapter = NPersQueue::NTests::TSimpleWriteSessionTestAdapter( + dynamic_cast(session.get())); + if (shouldCaptureData.has_value()) { + std::ostringstream msg; + msg << "Session has captured " << sessionAdapter.GetAcquiredMessagesCount() + << " messages, capturing was expected: " << *shouldCaptureData << std::endl; + ASSERT_TRUE(sessionAdapter.GetAcquiredMessagesCount() > 0) << msg.str(); + } + } + session->Close(); + + std::shared_ptr ReadSession; + + // Create topic client. + TTopicClient topicClient(driver); + + // Create read session. + TReadSessionSettings readSettings; + readSettings + .ConsumerName("test-consumer") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(GetTopicPath()); + + std::cerr << "Session was created" << std::endl; + + NThreading::TPromise checkedPromise = NThreading::NewPromise(); + auto totalReceived = 0u; + + auto f = checkedPromise.GetFuture(); + std::atomic check = 1; + readSettings.EventHandlers_.SimpleDataHandlers( + // [checkedPromise = std::move(checkedPromise), &check, &sentMessages, &totalReceived] + [&] + (TReadSessionEvent::TDataReceivedEvent& ev) mutable { + Y_ABORT_UNLESS(check.load() != 0, "check is false"); + auto& messages = ev.GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + ASSERT_EQ(message.GetData(), sentMessages[totalReceived]); + totalReceived++; + } + if (totalReceived == sentMessages.size()) + checkedPromise.SetValue(); + }); + + ReadSession = topicClient.CreateReadSession(readSettings); + + f.GetValueSync(); + ReadSession->Close(TDuration::MilliSeconds(10)); + check.store(0); + + auto status = topicClient.CommitOffset(GetTopicPath(), 0, "test-consumer", 50); + ASSERT_TRUE(status.GetValueSync().IsSuccess()); + + auto describeConsumerSettings = TDescribeConsumerSettings().IncludeStats(true); + auto result = topicClient.DescribeConsumer(GetTopicPath(), "test-consumer", describeConsumerSettings).GetValueSync(); + ASSERT_TRUE(result.IsSuccess()); + + auto description = result.GetConsumerDescription(); + ASSERT_EQ(description.GetPartitions().size(), 1u); + auto stats = description.GetPartitions().front().GetPartitionConsumerStats(); + ASSERT_TRUE(stats.has_value()); + ASSERT_EQ(stats->GetCommittedOffset(), 50u); +} + +TEST_F(BasicUsage, SessionNotDestroyedWhileCompressionInFlight) { + auto driver = MakeDriver(); + + // controlled executor + auto stepByStepExecutor = CreateThreadPoolManagedExecutor(1); + + // Create topic client. + TTopicClient topicClient(driver); + + NThreading::TPromise promiseToWrite = NThreading::NewPromise(); + auto futureWrite = promiseToWrite.GetFuture(); + + NThreading::TPromise promiseToRead = NThreading::NewPromise(); + auto futureRead = promiseToRead.GetFuture(); + + TWriteSessionSettings writeSettings; + writeSettings.Path(GetTopicPath()) + .MessageGroupId("test-message_group_id") + .ProducerId("test-message_group_id") + .CompressionExecutor(stepByStepExecutor); + + // Create read session. + TReadSessionSettings readSettings; + readSettings + .ConsumerName("test-consumer") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(GetTopicPath()) + .DecompressionExecutor(stepByStepExecutor); + + auto f = std::async(std::launch::async, + [readSettings, writeSettings, &topicClient, + promiseToWrite = std::move(promiseToWrite), + promiseToRead = std::move(promiseToRead)]() mutable { + { + auto writeSession = topicClient.CreateSimpleBlockingWriteSession(writeSettings); + std::string message(2'000, 'x'); + bool res = writeSession->Write(message); + ASSERT_TRUE(res); + writeSession->Close(TDuration::Seconds(10)); + } + promiseToWrite.SetValue(); + std::cerr << ">>>TEST: write promise set " << std::endl; + + { + NThreading::TPromise promise = NThreading::NewPromise(); + auto future = promise.GetFuture(); + + readSettings.EventHandlers_.SimpleDataHandlers( + [promise = std::move(promise)](TReadSessionEvent::TDataReceivedEvent& ev) mutable { + ev.Commit(); + promise.SetValue(); + std::cerr << ">>>TEST: get read event " << std::endl; + }); + + auto readSession = topicClient.CreateReadSession(readSettings); + future.Wait(); + readSession->Close(TDuration::Seconds(10)); + } + promiseToRead.SetValue(); + std::cerr << ">>>TEST: read promise set " << std::endl; + }); + + + // + // auxiliary functions for decompressor and handler control + // + auto WaitTasks = [&](auto f, size_t c) { + while (f() < c) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + }; + }; + auto WaitPlannedTasks = [&](auto e, size_t count) { + WaitTasks([&]() { return e->GetPlannedCount(); }, count); + }; + auto WaitExecutedTasks = [&](auto e, size_t count) { + WaitTasks([&]() { return e->GetExecutedCount(); }, count); + }; + + auto RunTasks = [&](auto e, const std::vector& tasks) { + size_t n = tasks.size(); + std::cerr << ">>>TEST in RunTasks: before WaitPlannedTasks" << std::endl; + WaitPlannedTasks(e, n); + std::cerr << ">>>TEST in RunTasks: before WaitExecutedTasks" << std::endl; + size_t completed = e->GetExecutedCount(); + e->StartFuncs(tasks); + WaitExecutedTasks(e, completed + n); + }; + + ASSERT_FALSE(futureWrite.HasValue()); + std::cerr << ">>>TEST: future write has no value " << std::endl; + RunTasks(stepByStepExecutor, {0}); // Run compression task. + RunTasks(stepByStepExecutor, {1}); // Run send task. + futureWrite.GetValueSync(); + ASSERT_TRUE(futureWrite.HasValue()); + std::cerr << ">>>TEST: future write has value " << std::endl; + + ASSERT_FALSE(futureRead.HasValue()); + std::cerr << ">>>TEST: future read has no value " << std::endl; + RunTasks(stepByStepExecutor, {2}); // Run decompression task. + futureRead.GetValueSync(); + ASSERT_TRUE(futureRead.HasValue()); + std::cerr << ">>>TEST: future read has value " << std::endl; + + f.get(); + + std::cerr << ">>> TEST: gracefully closed" << std::endl; +} + +TEST_F(BasicUsage, SessionNotDestroyedWhileUserEventHandlingInFlight) { + auto driver = MakeDriver(); + + // controlled executor + auto stepByStepExecutor = CreateThreadPoolManagedExecutor(1); + + // Create topic client. + TTopicClient topicClient(driver); + + // NThreading::TPromise promiseToWrite = NThreading::NewPromise(); + // auto futureWrite = promiseToWrite.GetFuture(); + + NThreading::TPromise promiseToRead = NThreading::NewPromise(); + auto futureRead = promiseToRead.GetFuture(); + + auto writeSettings = TWriteSessionSettings() + .Path(GetTopicPath()) + .MessageGroupId("test-message_group_id") + .ProducerId("test-message_group_id"); + + auto writeSession = topicClient.CreateSimpleBlockingWriteSession(writeSettings); + std::string message(2'000, 'x'); + bool res = writeSession->Write(message); + ASSERT_TRUE(res); + writeSession->Close(TDuration::Seconds(10)); + + // writeSettings.EventHandlers_ + // .HandlersExecutor(stepByStepExecutor); + + // Create read session. + auto readSettings = TReadSessionSettings() + .ConsumerName("test-consumer") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(GetTopicPath()); + + readSettings.EventHandlers_ + .HandlersExecutor(stepByStepExecutor); + + auto f = std::async(std::launch::async, + [readSettings, /*writeSettings,*/ &topicClient, + // promiseToWrite = std::move(promiseToWrite), + promiseToRead = std::move(promiseToRead)]() mutable { + // { + // std::shared_ptr token; + // writeSettings.EventHandlers_.CommonHandler([token](TWriteSessionEvent::TEvent& event){ + // std::cerr << ">>>TEST: in CommonHandler " << std::endl; + + // if (std::holds_alternative(event)) { + // *token = std::move(std::get(event).ContinuationToken); + // } + // }); + + // auto writeSession = topicClient.CreateWriteSession(writeSettings); + // std::string message(2'000, 'x'); + // writeSession->WaitEvent().Wait(); + // writeSession->Write(std::move(*token), message); + // writeSession->WaitEvent().Wait(); + // writeSession->Close(TDuration::Seconds(10)); + // } + // promiseToWrite.SetValue(); + // std::cerr << ">>>TEST: write promise set " << std::endl; + + { + NThreading::TPromise promise = NThreading::NewPromise(); + auto future = promise.GetFuture(); + + readSettings.EventHandlers_.SimpleDataHandlers( + [promise = std::move(promise)](TReadSessionEvent::TDataReceivedEvent& ev) mutable { + std::cerr << ">>>TEST: in SimpleDataHandlers " << std::endl; + ev.Commit(); + promise.SetValue(); + }); + + auto readSession = topicClient.CreateReadSession(readSettings); + future.Wait(); + readSession->Close(TDuration::Seconds(10)); + } + promiseToRead.SetValue(); + std::cerr << ">>>TEST: read promise set " << std::endl; + }); + + + // + // auxiliary functions for decompressor and handler control + // + auto WaitTasks = [&](auto f, size_t c) { + while (f() < c) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + }; + }; + auto WaitPlannedTasks = [&](auto e, size_t count) { + WaitTasks([&]() { return e->GetPlannedCount(); }, count); + }; + auto WaitExecutedTasks = [&](auto e, size_t count) { + WaitTasks([&]() { return e->GetExecutedCount(); }, count); + }; + + auto RunTasks = [&](auto e, const std::vector& tasks) { + size_t n = tasks.size(); + std::cerr << ">>>TEST in RunTasks: before WaitPlannedTasks" << std::endl; + WaitPlannedTasks(e, n); + std::cerr << ">>>TEST in RunTasks: before WaitExecutedTasks" << std::endl; + size_t completed = e->GetExecutedCount(); + e->StartFuncs(tasks); + WaitExecutedTasks(e, completed + n); + }; + + // RunTasks(stepByStepExecutor, {0}); + // UNIT_ASSERT(!futureWrite.HasValue()); + // std::cerr << ">>>TEST: future write has no value " << std::endl; + // RunTasks(stepByStepExecutor, {1}); + // futureWrite.GetValueSync(); + // UNIT_ASSERT(futureWrite.HasValue()); + // std::cerr << ">>>TEST: future write has value " << std::endl; + + ASSERT_FALSE(futureRead.HasValue()); + std::cerr << ">>>TEST: future read has no value " << std::endl; + // 0: TStartPartitionSessionEvent + RunTasks(stepByStepExecutor, {0}); + // 1: TDataReceivedEvent + RunTasks(stepByStepExecutor, {1}); + futureRead.GetValueSync(); + ASSERT_TRUE(futureRead.HasValue()); + std::cerr << ">>>TEST: future read has value " << std::endl; + + f.get(); + + std::cerr << ">>> TEST: gracefully closed" << std::endl; +} + +TEST_F(BasicUsage, ReadSessionCorrectClose) { + auto driver = MakeDriver(); + + NPersQueue::TWriteSessionSettings writeSettings; + writeSettings.Path(GetTopicPath()).MessageGroupId("src_id"); + writeSettings.Codec(NPersQueue::ECodec::RAW); + IExecutor::TPtr executor = new TSyncExecutor(); + writeSettings.CompressionExecutor(executor); + + NPersQueue::TPersQueueClient client(driver); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + + std::uint32_t count = 7000; + std::string message(2'000, 'x'); + for (std::uint32_t i = 1; i <= count; ++i) { + bool res = session->Write(message); + ASSERT_TRUE(res); + } + bool res = session->Close(TDuration::Seconds(30)); + ASSERT_TRUE(res); + + std::shared_ptr ReadSession; + + // Create topic client. + TTopicClient topicClient(driver); + + // Create read session. + NYdb::NTopic::TReadSessionSettings readSettings; + readSettings + .ConsumerName("test-consumer") + .MaxMemoryUsageBytes(1_MB) + .Decompress(false) + .RetryPolicy(NYdb::NTopic::IRetryPolicy::GetNoRetryPolicy()) + .AppendTopics(GetTopicPath()); + + readSettings.EventHandlers_.SimpleDataHandlers( + [] + (NYdb::NTopic::TReadSessionEvent::TDataReceivedEvent& ev) mutable { + std::cerr << ">>> Got TDataReceivedEvent" << std::endl; + ev.Commit(); + }); + + std::cerr << ">>> TEST: Create session" << std::endl; + + ReadSession = topicClient.CreateReadSession(readSettings); + + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + + ReadSession->Close(); + ReadSession = nullptr; + std::cerr << ">>> TEST: Session gracefully closed" << std::endl; + + std::this_thread::sleep_for(std::chrono::seconds(5)); +} + +TEST_F(BasicUsage, ConfirmPartitionSessionWithCommitOffset) { + // TStartPartitionSessionEvent::Confirm(readOffset, commitOffset) should work, + // if commitOffset passed to Confirm is greater than the offset committed previously by the consumer. + // https://st.yandex-team.ru/KIKIMR-23015 + + auto driver = MakeDriver(); + + { + // Write 2 messages: + auto settings = NTopic::TWriteSessionSettings() + .Path(GetTopicPath()) + .MessageGroupId("test-message_group_id") + .ProducerId("test-message_group_id"); + TTopicClient client(driver); + auto writer = client.CreateSimpleBlockingWriteSession(settings); + writer->Write("message"); + writer->Write("message"); + writer->Close(); + } + + { + // Read messages: + auto settings = NTopic::TReadSessionSettings() + .ConsumerName("test-consumer") + .AppendTopics(GetTopicPath()); + + TTopicClient client(driver); + auto reader = client.CreateReadSession(settings); + + { + // Start partition session and request to read from offset 1 and commit offset 1: + auto event = reader->GetEvent(true); + ASSERT_TRUE(event.has_value()); + ASSERT_TRUE(std::holds_alternative(*event)); + auto& startPartitionSession = std::get(*event); + startPartitionSession.Confirm(/*readOffset=*/ 1, /*commitOffset=*/ 1); + } + + { + // Receive a message with offset 1 and commit it: + auto event = reader->GetEvent(true); + ASSERT_TRUE(event.has_value()); + ASSERT_TRUE(std::holds_alternative(*event)); + auto& dataReceived = std::get(*event); + + // Here we should commit range [1, 2), not [0, 2): + dataReceived.Commit(); + } + + { + // And then get a TCommitOffsetAcknowledgementEvent: + auto event = reader->GetEvent(true); + ASSERT_TRUE(event.has_value()); + ASSERT_TRUE(std::holds_alternative(*event)); + } + } +} + +TEST_F(BasicUsage, TWriteSession_WriteEncoded) { + // This test was adapted from ydb_persqueue tests. + // It writes 4 messages: 2 with default codec, 1 with explicitly set GZIP codec, 1 with RAW codec. + // The last message MUST be sent in a separate WriteRequest, as it has a codec field applied for all messages in the request. + // This separation currently happens in TWriteSessionImpl::SendImpl method. + + auto driver = MakeDriver(); + + TTopicClient client(driver); + + auto settings = TWriteSessionSettings() + .Path(GetTopicPath()) + .MessageGroupId("test-message_group_id"); + + size_t batchSize = 100000000; + settings.BatchFlushInterval(TDuration::Seconds(1000)); // Batch on size, not on time. + settings.BatchFlushSizeBytes(batchSize); + auto writer = client.CreateWriteSession(settings); + std::string message = "message"; + TString packed; + { + TStringOutput so(packed); + TZLibCompress oss(&so, ZLib::GZip, 6); + oss << message; + } + + std::cerr << message << " " << packed << "\n"; + + { + auto event = *writer->GetEvent(true); + ASSERT_FALSE(writer->WaitEvent().Wait(TDuration::Seconds(1))); + auto ev = writer->WaitEvent(); + ASSERT_TRUE(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->Write(std::move(continueToken), message); + ASSERT_TRUE(ev.Wait(TDuration::Seconds(1))); + } + { + auto event = *writer->GetEvent(true); + ASSERT_TRUE(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->Write(std::move(continueToken), ""); + } + { + auto event = *writer->GetEvent(true); + ASSERT_TRUE(std::holds_alternative(event)); + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->WriteEncoded(std::move(continueToken), packed, ECodec::GZIP, message.size()); + } + + std::uint32_t acks = 0, tokens = 0; + while(acks < 4 || tokens < 2) { + auto event = *writer->GetEvent(true); + if (std::holds_alternative(event)) { + acks += std::get(event).Acks.size(); + } + if (std::holds_alternative(event)) { + if (tokens == 0) { + auto continueToken = std::move(std::get(event).ContinuationToken); + writer->WriteEncoded(std::move(continueToken), "", ECodec::RAW, 0); + } + ++tokens; + } + std::cerr << "GOT EVENT " << acks << " " << tokens << "\n"; + } + ASSERT_FALSE(writer->WaitEvent().Wait(TDuration::Seconds(5))); + + ASSERT_EQ(acks, 4u); + ASSERT_EQ(tokens, 2u); + + auto readSettings = TReadSessionSettings() + .ConsumerName("test-consumer") + .AppendTopics(GetTopicPath()); + std::shared_ptr readSession = client.CreateReadSession(readSettings); + std::uint32_t readMessageCount = 0; + while (readMessageCount < 4) { + std::cerr << "Get event on client\n"; + auto event = *readSession->GetEvent(true); + std::visit(TOverloaded { + [&](TReadSessionEvent::TDataReceivedEvent& event) { + for (auto& message: event.GetMessages()) { + std::string sourceId = message.GetMessageGroupId(); + std::uint32_t seqNo = message.GetSeqNo(); + ASSERT_EQ(readMessageCount + 1, seqNo); + ++readMessageCount; + ASSERT_EQ(message.GetData(), (seqNo % 2) == 1 ? "message" : ""); + } + }, + [&](TReadSessionEvent::TCommitOffsetAcknowledgementEvent&) { + FAIL(); + }, + [&](TReadSessionEvent::TStartPartitionSessionEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TStopPartitionSessionEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TEndPartitionSessionEvent& event) { + event.Confirm(); + }, + [&](TReadSessionEvent::TPartitionSessionStatusEvent&) { + FAIL() << "Test does not support lock sessions yet"; + }, + [&](TReadSessionEvent::TPartitionSessionClosedEvent&) { + FAIL() << "Test does not support lock sessions yet"; + }, + [&](TSessionClosedEvent&) { + FAIL() << "Session closed"; + } + + }, event); + } +} + +namespace { + enum class EExpectedTestResult { + SUCCESS, + FAIL_ON_SDK, + FAIL_ON_RPC + }; +} + +class TSettingsValidation : public TTopicTestFixture {}; + +TEST_F(TSettingsValidation, TestDifferentDedupParams) { + char* ydbVersion = std::getenv("YDB_VERSION"); + if (ydbVersion != nullptr && std::string(ydbVersion) != "trunk" && std::string(ydbVersion) < "24.3") { + GTEST_SKIP() << "Skipping test for YDB version " << ydbVersion; + } + + auto driver = MakeDriver(); + + TTopicClient client(driver); + + std::uint64_t producerIndex = 0u; + auto runTest = [&](std::string producer, std::string msgGroup, const std::optional& useDedup, bool useSeqNo, EExpectedTestResult result) { + TWriteSessionSettings writeSettings; + writeSettings.Path(GetTopicPath()).Codec(NTopic::ECodec::RAW); + std::string useDedupStr = useDedup.has_value() ? ToString(*useDedup) : ""; + if (!producer.empty()) { + producer += ToString(producerIndex); + } + if (!msgGroup.empty()) { + msgGroup += ToString(producerIndex); + } + writeSettings.ProducerId(producer).MessageGroupId(msgGroup); + producerIndex++; + std::cerr.flush(); + std::this_thread::sleep_for(std::chrono::milliseconds(250)); + std::cerr << "=== === START TEST. Producer = '" << producer << "', MsgGroup = '" << msgGroup << "', useDedup: " + << useDedupStr << ", manual SeqNo: " << useSeqNo << std::endl; + + try { + if (useDedup.has_value()) { + writeSettings.DeduplicationEnabled(useDedup); + } + auto session = client.CreateWriteSession(writeSettings); + std::optional token; + std::uint64_t seqNo = 1u; + std::uint64_t written = 0; + while (written < 10) { + auto event = session->GetEvent(true); + if (std::holds_alternative(event.value())) { + auto closed = std::get(*event); + std::cerr << "Session failed with error: " << closed.DebugString() << std::endl; + ASSERT_EQ(result, EExpectedTestResult::FAIL_ON_RPC); + return; + } else if (std::holds_alternative(event.value())) { + token = std::move(std::get(*event).ContinuationToken); + if (useSeqNo) { + session->Write(std::move(*token), "data", seqNo++); + } else { + session->Write(std::move(*token), "data"); + } + continue; + } else { + ASSERT_TRUE(std::holds_alternative(*event)); + const auto& acks = std::get(*event); + for (const auto& ack : acks.Acks) { + ASSERT_EQ(ack.State, TWriteSessionEvent::TWriteAck::EES_WRITTEN); + } + written += acks.Acks.size(); + } + } + } catch(const NYdb::TContractViolation& ex) { + std::cerr << "Test fails on contract validation: " << ex.what() << std::endl; + ASSERT_EQ(result, EExpectedTestResult::FAIL_ON_SDK); + return; + } + std::cerr << "=== === END TEST (supposed ok)=== ===\n\n"; + ASSERT_EQ(result, EExpectedTestResult::SUCCESS); + }; + // Normal scenarios: + // Most common: + std::vector producers = {"producer", ""}; + std::vector> messageGroup = {std::nullopt, "producer", "messageGroup", ""}; + std::vector> useDedupVariants = {std::nullopt, true, false}; + std::vector manSeqNoVariants = {true, false}; + runTest("producer", {}, {}, false, EExpectedTestResult::SUCCESS); + runTest("producer", {}, {}, true, EExpectedTestResult::SUCCESS); + // Enable dedup (doesnt take affect anything as it is enabled anyway) + runTest("producer", {}, true, true, EExpectedTestResult::SUCCESS); + runTest("producer", {}, true, false, EExpectedTestResult::SUCCESS); + + //No producer, do dedup + runTest({}, {}, {}, false, EExpectedTestResult::SUCCESS); + // manual seqNo with no-dedup - error + runTest({}, {}, {}, true, EExpectedTestResult::FAIL_ON_SDK); + // No producer but do enable dedup + runTest({}, {}, true, true, EExpectedTestResult::SUCCESS); + runTest({}, {}, true, false, EExpectedTestResult::SUCCESS); + + // MsgGroup = producer with explicit dedup enabling or not + runTest("producer", "producer", {}, false, EExpectedTestResult::SUCCESS); + runTest("producer", "producer", {}, true, EExpectedTestResult::SUCCESS); + runTest("producer", "producer", true, true, EExpectedTestResult::SUCCESS); + runTest("producer", "producer", true, false, EExpectedTestResult::SUCCESS); + + //Bad scenarios + // MsgGroup != producer, triggers error + runTest("producer", "msgGroup", {}, false, EExpectedTestResult::FAIL_ON_SDK); + runTest("producer", "msgGroup", {}, true, EExpectedTestResult::FAIL_ON_SDK); + runTest("producer", "msgGroup", true, true, EExpectedTestResult::FAIL_ON_SDK); + runTest("producer", "msgGroup", true, false, EExpectedTestResult::FAIL_ON_SDK); + + //Set producer or msgGroupId but disnable dedup: + runTest("producer", {}, false, true, EExpectedTestResult::FAIL_ON_SDK); + runTest("producer", {}, false, false, EExpectedTestResult::FAIL_ON_SDK); + runTest({}, "msgGroup", false, true, EExpectedTestResult::FAIL_ON_SDK); + runTest({}, "msgGroup", false, false, EExpectedTestResult::FAIL_ON_SDK); + + //Use msgGroupId as producerId, enable dedup + runTest({}, "msgGroup", true, true, EExpectedTestResult::SUCCESS); + runTest({}, "msgGroup", true, false, EExpectedTestResult::SUCCESS); + + + //Specify msg groupId and don't specify deduplication. Should work with dedup enable + runTest({}, "msgGroup", {}, true, EExpectedTestResult::SUCCESS); + runTest({}, "msgGroup", {}, false, EExpectedTestResult::SUCCESS); +} + +TEST_F(TSettingsValidation, ValidateSettingsFailOnStart) { + auto driver = MakeDriver(); + + TTopicClient client(driver); + + auto readSettings = TReadSessionSettings() + .ConsumerName("test-consumer") + .MaxMemoryUsageBytes(0) + .AppendTopics(GetTopicPath()); + + auto readSession = client.CreateReadSession(readSettings); + auto event = readSession->GetEvent(true); + ASSERT_TRUE(event.has_value()); + + auto& closeEvent = std::get(*event); + ASSERT_NE(closeEvent.DebugString().find("Too small max memory usage"), std::string::npos); +} + +} // namespace