diff --git a/include/ydb-cpp-sdk/client/datastreams/datastreams.h b/include/ydb-cpp-sdk/client/datastreams/datastreams.h index 15ead26b0c..0e031374e5 100644 --- a/include/ydb-cpp-sdk/client/datastreams/datastreams.h +++ b/include/ydb-cpp-sdk/client/datastreams/datastreams.h @@ -101,13 +101,158 @@ namespace NYdb::NDataStreams::V1 { std::string ExplicitHashDecimal; }; + enum class EAutoPartitioningStrategy: uint32_t { + Unspecified = 0, + Disabled = 1, + ScaleUp = 2, + ScaleUpAndDown = 3, + Paused = 4, + }; + + struct TCreateStreamSettings; + struct TUpdateStreamSettings; + + + template + struct TPartitioningSettingsBuilder; + template + struct TAutoPartitioningSettingsBuilder; + + struct TAutoPartitioningSettings { + friend struct TAutoPartitioningSettingsBuilder; + friend struct TAutoPartitioningSettingsBuilder; + public: + TAutoPartitioningSettings() + : Strategy_(EAutoPartitioningStrategy::Disabled) + , StabilizationWindow_(TDuration::Seconds(0)) + , DownUtilizationPercent_(0) + , UpUtilizationPercent_(0) { + } + TAutoPartitioningSettings(const Ydb::DataStreams::V1::AutoPartitioningSettings& settings); + TAutoPartitioningSettings(EAutoPartitioningStrategy strategy, TDuration stabilizationWindow, uint64_t downUtilizationPercent, uint64_t upUtilizationPercent) + : Strategy_(strategy) + , StabilizationWindow_(stabilizationWindow) + , DownUtilizationPercent_(downUtilizationPercent) + , UpUtilizationPercent_(upUtilizationPercent) {} + + EAutoPartitioningStrategy GetStrategy() const { return Strategy_; }; + TDuration GetStabilizationWindow() const { return StabilizationWindow_; }; + uint32_t GetDownUtilizationPercent() const { return DownUtilizationPercent_; }; + uint32_t GetUpUtilizationPercent() const { return UpUtilizationPercent_; }; + private: + EAutoPartitioningStrategy Strategy_; + TDuration StabilizationWindow_; + uint32_t DownUtilizationPercent_; + uint32_t UpUtilizationPercent_; + }; + + + class TPartitioningSettings { + using TSelf = TPartitioningSettings; + friend struct TPartitioningSettingsBuilder; + friend struct TPartitioningSettingsBuilder; + public: + TPartitioningSettings() : MinActivePartitions_(0), MaxActivePartitions_(0), AutoPartitioningSettings_(){} + TPartitioningSettings(const Ydb::DataStreams::V1::PartitioningSettings& settings); + TPartitioningSettings(uint64_t minActivePartitions, uint64_t maxActivePartitions, TAutoPartitioningSettings autoscalingSettings = {}) + : MinActivePartitions_(minActivePartitions) + , MaxActivePartitions_(maxActivePartitions) + , AutoPartitioningSettings_(autoscalingSettings) { + } + + uint64_t GetMinActivePartitions() const { return MinActivePartitions_; }; + uint64_t GetMaxActivePartitions() const { return MaxActivePartitions_; }; + TAutoPartitioningSettings GetAutoPartitioningSettings() const { return AutoPartitioningSettings_; }; + private: + uint64_t MinActivePartitions_; + uint64_t MaxActivePartitions_; + TAutoPartitioningSettings AutoPartitioningSettings_; + }; + struct TCreateStreamSettings : public NYdb::TOperationRequestSettings { FLUENT_SETTING(uint32_t, ShardCount); FLUENT_SETTING_OPTIONAL(uint32_t, RetentionPeriodHours); FLUENT_SETTING_OPTIONAL(uint32_t, RetentionStorageMegabytes); FLUENT_SETTING(uint64_t, WriteQuotaKbPerSec); FLUENT_SETTING_OPTIONAL(EStreamMode, StreamMode); + + + FLUENT_SETTING_OPTIONAL(TPartitioningSettings, PartitioningSettings); + TPartitioningSettingsBuilder BeginConfigurePartitioningSettings(); }; + + template + struct TAutoPartitioningSettingsBuilder { + using TSelf = TAutoPartitioningSettingsBuilder; + public: + TAutoPartitioningSettingsBuilder(TPartitioningSettingsBuilder& parent, TAutoPartitioningSettings& settings): Parent_(parent), Settings_(settings) {} + + TSelf Strategy(EAutoPartitioningStrategy value) { + Settings_.Strategy_ = value; + return *this; + } + + TSelf StabilizationWindow(TDuration value) { + Settings_.StabilizationWindow_ = value; + return *this; + } + + TSelf DownUtilizationPercent(uint32_t value) { + Settings_.DownUtilizationPercent_ = value; + return *this; + } + + TSelf UpUtilizationPercent(uint32_t value) { + Settings_.UpUtilizationPercent_ = value; + return *this; + } + + TPartitioningSettingsBuilder& EndConfigureAutoPartitioningSettings() { + return Parent_; + } + + private: + TPartitioningSettingsBuilder& Parent_; + TAutoPartitioningSettings& Settings_; + }; + + template + struct TPartitioningSettingsBuilder { + using TSelf = TPartitioningSettingsBuilder; + public: + TPartitioningSettingsBuilder(TSettings& parent): Parent_(parent) {} + + TSelf MinActivePartitions(uint64_t value) { + if (!Parent_.PartitioningSettings_.has_value()) { + Parent_.PartitioningSettings_.emplace(); + } + (*Parent_.PartitioningSettings_).MinActivePartitions_ = value; + return *this; + } + + TSelf MaxActivePartitions(uint64_t value) { + if (!Parent_.PartitioningSettings_.has_value()) { + Parent_.PartitioningSettings_.emplace(); + } + (*Parent_.PartitioningSettings_).MaxActivePartitions_ = value; + return *this; + } + + TAutoPartitioningSettingsBuilder BeginConfigureAutoPartitioningSettings() { + if (!Parent_.PartitioningSettings_.has_value()) { + Parent_.PartitioningSettings_.emplace(); + } + return {*this, (*Parent_.PartitioningSettings_).AutoPartitioningSettings_}; + } + + TSettings& EndConfigurePartitioningSettings() { + return Parent_; + } + + private: + TSettings& Parent_; + }; + struct TListStreamsSettings : public NYdb::TOperationRequestSettings { FLUENT_SETTING(uint32_t, Limit); FLUENT_SETTING(std::string, ExclusiveStartStreamName); @@ -155,6 +300,8 @@ namespace NYdb::NDataStreams::V1 { FLUENT_SETTING(uint64_t, WriteQuotaKbPerSec); FLUENT_SETTING_OPTIONAL(EStreamMode, StreamMode); + FLUENT_SETTING_OPTIONAL(TPartitioningSettings, PartitioningSettings); + TPartitioningSettingsBuilder BeginConfigurePartitioningSettings(); }; struct TPutRecordSettings : public NYdb::TOperationRequestSettings {}; struct TPutRecordsSettings : public NYdb::TOperationRequestSettings {}; diff --git a/include/ydb-cpp-sdk/client/draft/ydb_replication.h b/include/ydb-cpp-sdk/client/draft/ydb_replication.h index 4ae8019484..7528395245 100644 --- a/include/ydb-cpp-sdk/client/draft/ydb_replication.h +++ b/include/ydb-cpp-sdk/client/draft/ydb_replication.h @@ -5,9 +5,12 @@ #include +#include + namespace Ydb::Replication { class ConnectionParams; class DescribeReplicationResult; + class DescribeReplicationResult_Stats; } namespace NYdb { @@ -22,7 +25,11 @@ namespace NYdb::NReplication { class TDescribeReplicationResult; using TAsyncDescribeReplicationResult = NThreading::TFuture; -struct TDescribeReplicationSettings: public TOperationRequestSettings {}; + +struct TDescribeReplicationSettings: public TOperationRequestSettings { + using TSelf = TDescribeReplicationSettings; + FLUENT_SETTING_DEFAULT(bool, IncludeStats, false); +}; struct TStaticCredentials { std::string User; @@ -44,6 +51,7 @@ class TConnectionParams: private TCommonClientSettings { const std::string& GetDiscoveryEndpoint() const; const std::string& GetDatabase() const; + bool GetEnableSsl() const; ECredentials GetCredentials() const; const TStaticCredentials& GetStaticCredentials() const; @@ -56,7 +64,30 @@ class TConnectionParams: private TCommonClientSettings { > Credentials_; }; -struct TRunningState {}; +class TStats { +public: + TStats() = default; + TStats(const Ydb::Replication::DescribeReplicationResult_Stats& stats); + + const std::optional& GetLag() const; + const std::optional& GetInitialScanProgress() const; + +private: + std::optional Lag_; + std::optional InitialScanProgress_; +}; + +class TRunningState { +public: + TRunningState() = default; + explicit TRunningState(const TStats& stats); + + const TStats& GetStats() const; + +private: + TStats Stats_; +}; + struct TDoneState {}; class TErrorState { @@ -77,6 +108,7 @@ class TReplicationDescription { uint64_t Id; std::string SrcPath; std::string DstPath; + TStats Stats; std::optional SrcChangefeedName; }; diff --git a/include/ydb-cpp-sdk/client/query/client.h b/include/ydb-cpp-sdk/client/query/client.h index dbf611825b..a25f83531a 100644 --- a/include/ydb-cpp-sdk/client/query/client.h +++ b/include/ydb-cpp-sdk/client/query/client.h @@ -14,7 +14,11 @@ namespace NYdb { namespace NRetry::Async { template class TRetryContext; - } + } // namespace NRetry::Async + namespace NRetry::Sync { + template + class TRetryContext; + } // namespace NRetry::Sync } namespace NYdb::NQuery { @@ -55,10 +59,15 @@ class TSession; class TQueryClient { friend class TSession; friend class NRetry::Async::TRetryContext; + friend class NRetry::Async::TRetryContext; + friend class NRetry::Sync::TRetryContext; public: - using TQueryFunc = std::function; - using TQueryWithoutSessionFunc = std::function; + using TQueryResultFunc = std::function; + using TQueryFunc = std::function; + using TQuerySyncFunc = std::function; + using TQueryWithoutSessionFunc = std::function; + using TQueryWithoutSessionSyncFunc = std::function; using TSettings = TClientSettings; using TSession = TSession; using TCreateSessionSettings = TCreateSessionSettings; @@ -79,7 +88,15 @@ class TQueryClient { TAsyncExecuteQueryIterator StreamExecuteQuery(const std::string& query, const TTxControl& txControl, const TParams& params, const TExecuteQuerySettings& settings = TExecuteQuerySettings()); - TAsyncExecuteQueryResult RetryQuery(TQueryFunc&& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + TAsyncExecuteQueryResult RetryQuery(TQueryResultFunc&& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TAsyncStatus RetryQuery(TQueryFunc&& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TAsyncStatus RetryQuery(TQueryWithoutSessionFunc&& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TStatus RetryQuery(const TQuerySyncFunc& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); + + TStatus RetryQuery(const TQueryWithoutSessionSyncFunc& queryFunc, TRetryOperationSettings settings = TRetryOperationSettings()); TAsyncExecuteQueryResult RetryQuery(const std::string& query, const TTxControl& txControl, TDuration timeout, bool isIndempotent); diff --git a/include/ydb-cpp-sdk/client/query/query.h b/include/ydb-cpp-sdk/client/query/query.h index bdb01e5b79..7a07eb53fc 100644 --- a/include/ydb-cpp-sdk/client/query/query.h +++ b/include/ydb-cpp-sdk/client/query/query.h @@ -74,6 +74,7 @@ struct TExecuteQuerySettings : public TRequestSettings { FLUENT_SETTING_DEFAULT(EExecMode, ExecMode, EExecMode::Execute); FLUENT_SETTING_DEFAULT(EStatsMode, StatsMode, EStatsMode::None); FLUENT_SETTING_OPTIONAL(bool, ConcurrentResultSets); + FLUENT_SETTING(std::string, PoolId); }; struct TBeginTxSettings : public TRequestSettings {}; @@ -97,6 +98,7 @@ struct TExecuteScriptSettings : public TOperationRequestSettings PermissionNames; + + void SerializeTo(::Ydb::Scheme::Permissions& proto) const; }; enum class ESchemeEntryType : i32 { @@ -42,7 +46,8 @@ enum class ESchemeEntryType : i32 { Topic = 17, ExternalTable = 18, ExternalDataSource = 19, - View = 20 + View = 20, + ResourcePool = 21, }; struct TVirtualTimestamp { @@ -77,6 +82,9 @@ struct TSchemeEntry { TSchemeEntry(const ::Ydb::Scheme::Entry& proto); void Out(IOutputStream& out) const; + + // Fills ModifyPermissionsRequest proto from this entry + void SerializeTo(::Ydb::Scheme::ModifyPermissionsRequest& request) const; }; //////////////////////////////////////////////////////////////////////////////// diff --git a/include/ydb-cpp-sdk/client/table/table.h b/include/ydb-cpp-sdk/client/table/table.h index 245b9f1742..82ae8c7291 100644 --- a/include/ydb-cpp-sdk/client/table/table.h +++ b/include/ydb-cpp-sdk/client/table/table.h @@ -21,6 +21,9 @@ class CreateTableRequest; class Changefeed; class ChangefeedDescription; class DescribeTableResult; +class ExplicitPartitions; +class GlobalIndexSettings; +class VectorIndexSettings; class PartitioningSettings; class DateTypeColumnModeSettings; class TtlSettings; @@ -146,22 +149,113 @@ struct TAlterTableColumn { //////////////////////////////////////////////////////////////////////////////// +//! Represents table partitioning settings +class TPartitioningSettings { +public: + TPartitioningSettings(); + explicit TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto); + + const Ydb::Table::PartitioningSettings& GetProto() const; + + std::optional GetPartitioningBySize() const; + std::optional GetPartitioningByLoad() const; + uint64_t GetPartitionSizeMb() const; + uint64_t GetMinPartitionsCount() const; + uint64_t GetMaxPartitionsCount() const; + +private: + class TImpl; + std::shared_ptr Impl_; +}; + +struct TExplicitPartitions { + using TSelf = TExplicitPartitions; + + FLUENT_SETTING_VECTOR(TValue, SplitPoints); + + template + static TExplicitPartitions FromProto(const TProto& proto); + + void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const; +}; + +struct TGlobalIndexSettings { + using TUniformOrExplicitPartitions = std::variant; + + TPartitioningSettings PartitioningSettings; + TUniformOrExplicitPartitions Partitions; + + template + static TGlobalIndexSettings FromProto(const TProto& proto); + + void SerializeTo(Ydb::Table::GlobalIndexSettings& proto) const; +}; + +struct TVectorIndexSettings { +public: + enum class EDistance { + Cosine, + Manhattan, + Euclidean, + + Unknown = std::numeric_limits::max() + }; + + enum class ESimilarity { + Cosine, + InnerProduct, + + Unknown = std::numeric_limits::max() + }; + + enum class EVectorType { + Float, + Uint8, + Int8, + Bit, + + Unknown = std::numeric_limits::max() + }; + using TMetric = std::variant; + + TMetric Metric; + EVectorType VectorType; + uint32_t VectorDimension; + + template + static TVectorIndexSettings FromProto(const TProto& proto); + + void SerializeTo(Ydb::Table::VectorIndexSettings& settings) const; + + void Out(IOutputStream &o) const; +}; + //! Represents index description class TIndexDescription { friend class NYdb::TProtoAccessor; public: TIndexDescription( - const std::string& name, EIndexType type, + const std::string& name, + EIndexType type, const std::vector& indexColumns, - const std::vector& dataColumns = std::vector()); + const std::vector& dataColumns = {}, + const std::vector& globalIndexSettings = {}, + const std::optional& vectorIndexSettings = {} + ); - TIndexDescription(const std::string& name, const std::vector& indexColumns, const std::vector& dataColumns = std::vector()); + TIndexDescription( + const std::string& name, + const std::vector& indexColumns, + const std::vector& dataColumns = {}, + const std::vector& globalIndexSettings = {} + ); const std::string& GetIndexName() const; EIndexType GetIndexType() const; const std::vector& GetIndexColumns() const; const std::vector& GetDataColumns() const; + const std::optional& GetVectorIndexSettings() const; uint64_t GetSizeBytes() const; void SerializeTo(Ydb::Table::TableIndex& proto) const; @@ -180,6 +274,8 @@ class TIndexDescription { EIndexType IndexType_; std::vector IndexColumns_; std::vector DataColumns_; + std::vector GlobalIndexSettings_; + std::optional VectorIndexSettings_; uint64_t SizeBytes = 0; }; @@ -211,10 +307,27 @@ class TBuildIndexOperation : public TOperation { //////////////////////////////////////////////////////////////////////////////// -//! Represents index description +//! Represents changefeed description class TChangefeedDescription { friend class NYdb::TProtoAccessor; +public: + class TInitialScanProgress { + public: + TInitialScanProgress(); + explicit TInitialScanProgress(uint32_t total, uint32_t completed); + + TInitialScanProgress& operator+=(const TInitialScanProgress& other); + + uint32_t GetPartsTotal() const; + uint32_t GetPartsCompleted() const; + float GetProgress() const; // percentage + + private: + uint32_t PartsTotal; + uint32_t PartsCompleted; + }; + public: TChangefeedDescription(const std::string& name, EChangefeedMode mode, EChangefeedFormat format); @@ -242,6 +355,7 @@ class TChangefeedDescription { bool GetInitialScan() const; const std::unordered_map& GetAttributes() const; const std::string& GetAwsRegion() const; + const std::optional& GetInitialScanProgress() const; void SerializeTo(Ydb::Table::Changefeed& proto) const; std::string ToString() const; @@ -265,6 +379,7 @@ class TChangefeedDescription { bool InitialScan_ = false; std::unordered_map Attributes_; std::string AwsRegion_; + std::optional InitialScanProgress_; }; bool operator==(const TChangefeedDescription& lhs, const TChangefeedDescription& rhs); @@ -423,25 +538,6 @@ class TColumnFamilyDescription { std::shared_ptr Impl_; }; -//! Represents table partitioning settings -class TPartitioningSettings { -public: - TPartitioningSettings(); - explicit TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto); - - const Ydb::Table::PartitioningSettings& GetProto() const; - - std::optional GetPartitioningBySize() const; - std::optional GetPartitioningByLoad() const; - uint64_t GetPartitionSizeMb() const; - uint64_t GetMinPartitionsCount() const; - uint64_t GetMaxPartitionsCount() const; - -private: - class TImpl; - std::shared_ptr Impl_; -}; - //! Represents table read replicas settings class TReadReplicasSettings { public: @@ -544,6 +640,7 @@ class TTableDescription { // common void AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns); void AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns); + void AddSecondaryIndex(const TIndexDescription& indexDescription); // sync void AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); void AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); @@ -553,6 +650,9 @@ class TTableDescription { // unique void AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); void AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + // vector KMeansTree + void AddVectorKMeansTreeSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const TVectorIndexSettings& vectorIndexSettings); + void AddVectorKMeansTreeSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TVectorIndexSettings& vectorIndexSettings); // default void AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); @@ -754,6 +854,7 @@ class TTableBuilder { TTableBuilder& SetPrimaryKeyColumn(const std::string& primaryKeyColumn); // common + TTableBuilder& AddSecondaryIndex(const TIndexDescription& indexDescription); TTableBuilder& AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns); TTableBuilder& AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns); TTableBuilder& AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::string& indexColumn); @@ -772,6 +873,10 @@ class TTableBuilder { TTableBuilder& AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); TTableBuilder& AddUniqueSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); + // vector KMeansTree + TTableBuilder& AddVectorKMeansTreeSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const TVectorIndexSettings& vectorIndexSettings); + TTableBuilder& AddVectorKMeansTreeSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TVectorIndexSettings& vectorIndexSettings); + // default TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns); TTableBuilder& AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns); @@ -1219,12 +1324,6 @@ struct TStoragePolicy { FLUENT_SETTING_VECTOR(TColumnFamilyPolicy, ColumnFamilies); }; -struct TExplicitPartitions { - using TSelf = TExplicitPartitions; - - FLUENT_SETTING_VECTOR(TValue, SplitPoints); -}; - struct TPartitioningPolicy { using TSelf = TPartitioningPolicy; diff --git a/include/ydb-cpp-sdk/client/table/table_enum.h b/include/ydb-cpp-sdk/client/table/table_enum.h index 25b57b005b..1660706f57 100644 --- a/include/ydb-cpp-sdk/client/table/table_enum.h +++ b/include/ydb-cpp-sdk/client/table/table_enum.h @@ -28,6 +28,7 @@ enum class EIndexType { GlobalSync, GlobalAsync, GlobalUnique, + GlobalVectorKMeansTree, Unknown = std::numeric_limits::max() }; diff --git a/include/ydb-cpp-sdk/client/topic/control_plane.h b/include/ydb-cpp-sdk/client/topic/control_plane.h index 3369f8c984..42e4b84052 100644 --- a/include/ydb-cpp-sdk/client/topic/control_plane.h +++ b/include/ydb-cpp-sdk/client/topic/control_plane.h @@ -33,6 +33,7 @@ enum class EAutoPartitioningStrategy: uint32_t { Disabled = 1, ScaleUp = 2, ScaleUpAndDown = 3, + Paused = 4, }; class TConsumer { @@ -144,14 +145,21 @@ class TPartitionInfo { const std::optional& GetPartitionConsumerStats() const; const std::optional& GetPartitionLocation() const; + const std::optional& GetFromBound() const; + const std::optional& GetToBound() const; + private: uint64_t PartitionId_; bool Active_; std::vector ChildPartitionIds_; std::vector ParentPartitionIds_; + std::optional PartitionStats_; std::optional PartitionConsumerStats_; std::optional PartitionLocation_; + + std::optional FromBound_; + std::optional ToBound_; }; struct TAlterPartitioningSettings; @@ -160,7 +168,7 @@ struct TAlterTopicSettings; struct TAutoPartitioningSettings { friend struct TAutoPartitioningSettingsBuilder; public: -TAutoPartitioningSettings() + TAutoPartitioningSettings() : Strategy_(EAutoPartitioningStrategy::Disabled) , StabilizationWindow_(TDuration::Seconds(0)) , DownUtilizationPercent_(0) @@ -186,8 +194,8 @@ TAutoPartitioningSettings() struct TAlterAutoPartitioningSettings { using TSelf = TAlterAutoPartitioningSettings; - public: - TAlterAutoPartitioningSettings(TAlterPartitioningSettings& parent): Parent_(parent) {} +public: + TAlterAutoPartitioningSettings(TAlterPartitioningSettings& parent): Parent_(parent) {} FLUENT_SETTING_OPTIONAL(EAutoPartitioningStrategy, Strategy); FLUENT_SETTING_OPTIONAL(TDuration, StabilizationWindow); @@ -196,8 +204,8 @@ struct TAlterAutoPartitioningSettings { TAlterPartitioningSettings& EndAlterAutoPartitioningSettings() { return Parent_; }; - private: - TAlterPartitioningSettings& Parent_; +private: + TAlterPartitioningSettings& Parent_; }; class TPartitioningSettings { @@ -206,11 +214,11 @@ class TPartitioningSettings { public: TPartitioningSettings() : MinActivePartitions_(0), MaxActivePartitions_(0), PartitionCountLimit_(0), AutoPartitioningSettings_(){} TPartitioningSettings(const Ydb::Topic::PartitioningSettings& settings); - TPartitioningSettings(ui64 minActivePartitions, ui64 maxActivePartitions, TAutoPartitioningSettings autoscalingSettings = {}) + TPartitioningSettings(uint64_t minActivePartitions, uint64_t maxActivePartitions, TAutoPartitioningSettings autoPartitioning = {}) : MinActivePartitions_(minActivePartitions) , MaxActivePartitions_(maxActivePartitions) , PartitionCountLimit_(0) - , AutoPartitioningSettings_(autoscalingSettings) + , AutoPartitioningSettings_(autoPartitioning) { } @@ -459,6 +467,11 @@ struct TConsumerSettings { return *this; } + TConsumerSettings& SetImportant(bool isImportant) { + Important_ = isImportant; + return *this; + } + TSettings& EndAddConsumer() { return Parent_; }; private: diff --git a/include/ydb-cpp-sdk/client/topic/write_events.h b/include/ydb-cpp-sdk/client/topic/write_events.h index 56413200cd..46d195f688 100644 --- a/include/ydb-cpp-sdk/client/topic/write_events.h +++ b/include/ydb-cpp-sdk/client/topic/write_events.h @@ -56,7 +56,8 @@ struct TWriteSessionEvent { enum EEventState { EES_WRITTEN, //! Successfully written. EES_ALREADY_WRITTEN, //! Skipped on SeqNo deduplication. - EES_DISCARDED //! In case of destruction of writer or retry policy discarded future retries in this writer. + EES_DISCARDED, //! In case of destruction of writer or retry policy discarded future retries in this writer. + EES_WRITTEN_IN_TX, //! Successfully written in tx. }; //! Details of successfully written message. struct TWrittenMessageDetails { diff --git a/include/ydb-cpp-sdk/client/types/operation/operation.h b/include/ydb-cpp-sdk/client/types/operation/operation.h index 722bf947a5..88d226d2aa 100644 --- a/include/ydb-cpp-sdk/client/types/operation/operation.h +++ b/include/ydb-cpp-sdk/client/types/operation/operation.h @@ -5,6 +5,7 @@ #include #include +#include #include namespace Ydb { @@ -31,6 +32,9 @@ class TOperation { const TOperationId& Id() const; bool Ready() const; const TStatus& Status() const; + TInstant CreateTime() const; + TInstant EndTime() const; + const std::string& CreatedBy() const; std::string ToString() const; std::string ToJsonString() const; @@ -46,4 +50,6 @@ class TOperation { using TAsyncOperation = NThreading::TFuture; +TInstant ProtoTimestampToInstant(const google::protobuf::Timestamp& timestamp); + } // namespace NYdb diff --git a/src/client/datastreams/datastreams.cpp b/src/client/datastreams/datastreams.cpp index e014026fb1..a17009b9b9 100644 --- a/src/client/datastreams/datastreams.cpp +++ b/src/client/datastreams/datastreams.cpp @@ -5,12 +5,50 @@ #undef INCLUDE_YDB_INTERNAL_H #include -//#include +#include #include namespace NYdb::NDataStreams::V1 { + TPartitioningSettingsBuilder TCreateStreamSettings::BeginConfigurePartitioningSettings() { + return { *this }; + } + + TPartitioningSettingsBuilder TUpdateStreamSettings::BeginConfigurePartitioningSettings() { + return { *this }; + } + + void SetPartitionSettings(const TPartitioningSettings& ps, ::Ydb::DataStreams::V1::PartitioningSettings* pt) { + pt->set_max_active_partitions(ps.GetMaxActivePartitions()); + pt->set_min_active_partitions(ps.GetMinActivePartitions()); + + ::Ydb::DataStreams::V1::AutoPartitioningStrategy strategy; + switch (ps.GetAutoPartitioningSettings().GetStrategy()) { + case EAutoPartitioningStrategy::Unspecified: + case EAutoPartitioningStrategy::Disabled: + strategy = ::Ydb::DataStreams::V1::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_DISABLED; + break; + case EAutoPartitioningStrategy::ScaleUp: + strategy = ::Ydb::DataStreams::V1::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_SCALE_UP; + break; + case EAutoPartitioningStrategy::ScaleUpAndDown: + strategy = ::Ydb::DataStreams::V1::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_SCALE_UP_AND_DOWN; + break; + case EAutoPartitioningStrategy::Paused: + strategy = ::Ydb::DataStreams::V1::AutoPartitioningStrategy::AUTO_PARTITIONING_STRATEGY_PAUSED; + break; + } + + pt->mutable_auto_partitioning_settings()->set_strategy(strategy); + pt->mutable_auto_partitioning_settings()->mutable_partition_write_speed() + ->mutable_stabilization_window()->set_seconds(ps.GetAutoPartitioningSettings().GetStabilizationWindow().Seconds()); + pt->mutable_auto_partitioning_settings()->mutable_partition_write_speed() + ->set_up_utilization_percent(ps.GetAutoPartitioningSettings().GetUpUtilizationPercent()); + pt->mutable_auto_partitioning_settings()->mutable_partition_write_speed() + ->set_down_utilization_percent(ps.GetAutoPartitioningSettings().GetDownUtilizationPercent()); + } + class TDataStreamsClient::TImpl : public TClientImplCommon { public: TImpl(std::shared_ptr &&connections, const TCommonClientSettings &settings) @@ -88,6 +126,10 @@ namespace NYdb::NDataStreams::V1 { *settings.StreamMode_ == ESM_PROVISIONED ? Ydb::DataStreams::V1::StreamMode::PROVISIONED : Ydb::DataStreams::V1::StreamMode::ON_DEMAND); } + + if (settings.PartitioningSettings_.has_value()) { + SetPartitionSettings(*settings.PartitioningSettings_, req.mutable_partitioning_settings()); + } }); } @@ -372,6 +414,10 @@ namespace NYdb::NDataStreams::V1 { *settings.StreamMode_ == ESM_PROVISIONED ? Ydb::DataStreams::V1::StreamMode::PROVISIONED : Ydb::DataStreams::V1::StreamMode::ON_DEMAND); } + + if (settings.PartitioningSettings_.has_value()) { + SetPartitionSettings(*settings.PartitioningSettings_, req.mutable_partitioning_settings()); + } }); } @@ -907,4 +953,3 @@ namespace NYdb::NDataStreams::V1 { TProtoRequestSettings settings ); } - diff --git a/src/client/discovery/discovery.cpp b/src/client/discovery/discovery.cpp index c7b04032ee..2295886e5c 100644 --- a/src/client/discovery/discovery.cpp +++ b/src/client/discovery/discovery.cpp @@ -120,7 +120,7 @@ uint64_t TNodeRegistrationResult::GetScopePathId() const { } bool TNodeRegistrationResult::HasScopePathId() const { - return ScopePathId_.value(); + return ScopePathId_.has_value(); } bool TNodeRegistrationResult::HasNodeName() const { diff --git a/src/client/draft/ydb_replication.cpp b/src/client/draft/ydb_replication.cpp index 8e7fccf96f..b2be0c7eed 100644 --- a/src/client/draft/ydb_replication.cpp +++ b/src/client/draft/ydb_replication.cpp @@ -10,6 +10,7 @@ #include #include +#include #include namespace NYdb { @@ -18,6 +19,7 @@ namespace NReplication { TConnectionParams::TConnectionParams(const Ydb::Replication::ConnectionParams& params) { DiscoveryEndpoint(params.endpoint()); Database(params.database()); + SslCredentials(params.enable_ssl()); switch (params.credentials_case()) { case Ydb::Replication::ConnectionParams::kStaticCredentials: @@ -46,6 +48,10 @@ const std::string& TConnectionParams::GetDatabase() const { return *Database_; } +bool TConnectionParams::GetEnableSsl() const { + return SslCredentials_->IsEnabled; +} + TConnectionParams::ECredentials TConnectionParams::GetCredentials() const { return static_cast(Credentials_.index()); } @@ -58,6 +64,33 @@ const TOAuthCredentials& TConnectionParams::GetOAuthCredentials() const { return std::get(Credentials_); } +static TDuration DurationToDuration(const google::protobuf::Duration& value) { + return TDuration::MilliSeconds(google::protobuf::util::TimeUtil::DurationToMilliseconds(value)); +} + +TStats::TStats(const Ydb::Replication::DescribeReplicationResult_Stats& stats) + : Lag_(stats.has_lag() ? std::make_optional(DurationToDuration(stats.lag())) : std::nullopt) + , InitialScanProgress_(stats.has_initial_scan_progress() ? std::make_optional(stats.initial_scan_progress()) : std::nullopt) +{ +} + +const std::optional& TStats::GetLag() const { + return Lag_; +} + +const std::optional& TStats::GetInitialScanProgress() const { + return InitialScanProgress_; +} + +TRunningState::TRunningState(const TStats& stats) + : Stats_(stats) +{ +} + +const TStats& TRunningState::GetStats() const { + return Stats_; +} + class TErrorState::TImpl { public: NYql::TIssues Issues; @@ -93,6 +126,7 @@ TReplicationDescription::TReplicationDescription(const Ydb::Replication::Describ .Id = item.id(), .SrcPath = item.source_path(), .DstPath = item.destination_path(), + .Stats = TStats(item.stats()), .SrcChangefeedName = item.has_source_changefeed_name() ? std::make_optional(item.source_changefeed_name()) : std::nullopt, }); @@ -100,7 +134,7 @@ TReplicationDescription::TReplicationDescription(const Ydb::Replication::Describ switch (desc.state_case()) { case Ydb::Replication::DescribeReplicationResult::kRunning: - State_ = TRunningState(); + State_ = TRunningState(desc.running().stats()); break; case Ydb::Replication::DescribeReplicationResult::kError: @@ -168,6 +202,7 @@ class TReplicationClient::TImpl: public TClientImplCommon(settings); request.set_path(TStringType{path}); + request.set_include_stats(settings.IncludeStats_); auto promise = NThreading::NewPromise(); diff --git a/src/client/export/export.cpp b/src/client/export/export.cpp index 44d952d889..8760903181 100644 --- a/src/client/export/export.cpp +++ b/src/client/export/export.cpp @@ -24,12 +24,6 @@ using namespace Ydb::Export; /// Common namespace { -TInstant ProtoTimestampToInstant(const google::protobuf::Timestamp& timestamp) { - ui64 us = timestamp.seconds() * 1000000; - us += timestamp.nanos() / 1000; - return TInstant::MicroSeconds(us); -} - std::vector ItemsProgressFromProto(const google::protobuf::RepeatedPtrField& proto) { std::vector result(proto.size()); diff --git a/src/client/federated_topic/impl/federated_topic_impl.cpp b/src/client/federated_topic/impl/federated_topic_impl.cpp index 3589ca236c..4cf28a6212 100644 --- a/src/client/federated_topic/impl/federated_topic_impl.cpp +++ b/src/client/federated_topic/impl/federated_topic_impl.cpp @@ -35,7 +35,8 @@ TFederatedTopicClient::TImpl::CreateWriteSession(const TFederatedWriteSessionSet splitSettings.EventHandlers_.HandlersExecutor(ClientSettings.DefaultHandlersExecutor_); } } - auto session = std::make_shared(splitSettings, Connections, ClientSettings, GetObserver(), ProvidedCodecs); + auto session = std::make_shared( + splitSettings, Connections, ClientSettings, GetObserver(), ProvidedCodecs, GetSubsessionHandlersExecutor()); session->Start(); return std::move(session); } @@ -48,4 +49,13 @@ void TFederatedTopicClient::TImpl::InitObserver() { } } +auto TFederatedTopicClient::TImpl::GetSubsessionHandlersExecutor() -> NTopic::IExecutor::TPtr { + with_lock (Lock) { + if (!SubsessionHandlersExecutor) { + SubsessionHandlersExecutor = NTopic::CreateThreadPoolExecutor(1); + } + return SubsessionHandlersExecutor; + } +} + } diff --git a/src/client/federated_topic/impl/federated_topic_impl.h b/src/client/federated_topic/impl/federated_topic_impl.h index 14991b4bdf..3c52904fbb 100644 --- a/src/client/federated_topic/impl/federated_topic_impl.h +++ b/src/client/federated_topic/impl/federated_topic_impl.h @@ -72,12 +72,20 @@ class TFederatedTopicClient::TImpl { void InitObserver(); +private: + + // Use single-threaded executor to prevent deadlocks inside subsession event handlers. + NTopic::IExecutor::TPtr GetSubsessionHandlersExecutor(); + private: std::shared_ptr Connections; const TFederatedTopicClientSettings ClientSettings; std::shared_ptr Observer; std::shared_ptr>> ProvidedCodecs = std::make_shared>>(); + + NTopic::IExecutor::TPtr SubsessionHandlersExecutor; + TAdaptiveLock Lock; }; diff --git a/src/client/federated_topic/impl/federated_write_session.cpp b/src/client/federated_topic/impl/federated_write_session.cpp index ca2ab80d21..dc33eb2aab 100644 --- a/src/client/federated_topic/impl/federated_write_session.cpp +++ b/src/client/federated_topic/impl/federated_write_session.cpp @@ -19,7 +19,7 @@ bool DatabasesAreSame(std::shared_ptr lhs, std::shared_ptr rhs if (!lhs || !rhs) { return false; } - return lhs->path() == rhs->path() && lhs->endpoint() == rhs->endpoint(); + return lhs->name() == rhs->name() && lhs->path() == rhs->path() && lhs->endpoint() == rhs->endpoint(); } NTopic::TTopicClientSettings FromFederated(const TFederatedTopicClientSettings& settings); @@ -29,12 +29,14 @@ TFederatedWriteSessionImpl::TFederatedWriteSessionImpl( std::shared_ptr connections, const TFederatedTopicClientSettings& clientSettings, std::shared_ptr observer, - std::shared_ptr>> codecs + std::shared_ptr>> codecs, + NTopic::IExecutor::TPtr subsessionHandlersExecutor ) : Settings(settings) , Connections(std::move(connections)) , SubclientSettings(FromFederated(clientSettings)) , ProvidedCodecs(std::move(codecs)) + , SubsessionHandlersExecutor(subsessionHandlersExecutor) , Observer(std::move(observer)) , AsyncInit(Observer->WaitForFirstState()) , FederationState(nullptr) @@ -70,15 +72,16 @@ void TFederatedWriteSessionImpl::IssueTokenIfAllowed() { } } -void TFederatedWriteSessionImpl::UpdateFederationStateImpl() { +std::shared_ptr TFederatedWriteSessionImpl::UpdateFederationStateImpl() { Y_ABORT_UNLESS(Lock.IsLocked()); // Even after the user has called the Close method, transitioning the session to the CLOSING state, // we keep updating the federation state, as the session may still have some messages to send in its queues, // and for that we need to know the current state of the federation. if (SessionState < State::CLOSED) { FederationState = Observer->GetState(); - OnFederationStateUpdateImpl(); + return OnFederationStateUpdateImpl(); } + return {}; } void TFederatedWriteSessionImpl::Start() { @@ -103,12 +106,18 @@ void TFederatedWriteSessionImpl::Start() { }); } -void TFederatedWriteSessionImpl::OpenSubsessionImpl(std::shared_ptr db) { +std::shared_ptr TFederatedWriteSessionImpl::OpenSubsessionImpl(std::shared_ptr db) { Y_ABORT_UNLESS(Lock.IsLocked()); + + ++SubsessionGeneration; + + std::shared_ptr oldSubsession; + if (Subsession) { PendingToken.reset(); - Subsession->Close(TDuration::Zero()); + std::swap(oldSubsession, Subsession); } + auto clientSettings = SubclientSettings; clientSettings .Database(db->path()) @@ -116,21 +125,27 @@ void TFederatedWriteSessionImpl::OpenSubsessionImpl(std::shared_ptr db) auto subclient = std::make_shared(Connections, clientSettings); auto handlers = NTopic::TWriteSessionSettings::TEventHandlers() - .HandlersExecutor(Settings.EventHandlers_.HandlersExecutor_) - .ReadyToAcceptHandler([selfCtx = SelfContext](NTopic::TWriteSessionEvent::TReadyToAcceptEvent& ev) { + .HandlersExecutor(SubsessionHandlersExecutor) + .ReadyToAcceptHandler([selfCtx = SelfContext, generation = SubsessionGeneration](NTopic::TWriteSessionEvent::TReadyToAcceptEvent& ev) { if (auto self = selfCtx->LockShared()) { - TDeferredWrite deferred(self->Subsession); with_lock(self->Lock) { + if (generation != self->SubsessionGeneration) { + return; + } + Y_ABORT_UNLESS(!self->PendingToken.has_value()); self->PendingToken = std::move(ev.ContinuationToken); - self->PrepareDeferredWriteImpl(deferred); + self->MaybeWriteImpl(); } - deferred.DoWrite(); } }) - .AcksHandler([selfCtx = SelfContext](NTopic::TWriteSessionEvent::TAcksEvent& ev) { + .AcksHandler([selfCtx = SelfContext, generation = SubsessionGeneration](NTopic::TWriteSessionEvent::TAcksEvent& ev) { if (auto self = selfCtx->LockShared()) { with_lock(self->Lock) { + if (generation != self->SubsessionGeneration) { + return; + } + Y_ABORT_UNLESS(ev.Acks.size() <= self->OriginalMessagesToGetAck.size()); for (size_t i = 0; i < ev.Acks.size(); ++i) { @@ -147,14 +162,30 @@ void TFederatedWriteSessionImpl::OpenSubsessionImpl(std::shared_ptr db) self->IssueTokenIfAllowed(); } }) - .SessionClosedHandler([selfCtx = SelfContext](const NTopic::TSessionClosedEvent & ev) { + .SessionClosedHandler([selfCtx = SelfContext, generation = SubsessionGeneration](const NTopic::TSessionClosedEvent & ev) { + if (ev.IsSuccess()) { + // The subsession was closed by the federated write session itself while creating a new subsession. + // In this case we get SUCCESS status and don't need to propagate it further. + return; + } if (auto self = selfCtx->LockShared()) { with_lock(self->Lock) { + if (generation != self->SubsessionGeneration) { + return; + } self->CloseImpl(ev); } } }); + { + // Unacknowledged messages should be resent. + for (auto& msg : OriginalMessagesToPassDown) { + OriginalMessagesToGetAck.emplace_back(std::move(msg)); + } + OriginalMessagesToPassDown = std::move(OriginalMessagesToGetAck); + } + NTopic::TWriteSessionSettings wsSettings = Settings; wsSettings // .MaxMemoryUsage(Settings.MaxMemoryUsage_) // to fix if split not by half on creation @@ -162,6 +193,8 @@ void TFederatedWriteSessionImpl::OpenSubsessionImpl(std::shared_ptr db) Subsession = subclient->CreateWriteSession(wsSettings); CurrentDatabase = db; + + return oldSubsession; } std::pair, EStatus> SelectDatabaseByHashImpl( @@ -246,13 +279,13 @@ std::pair, EStatus> SelectDatabaseImpl( return SelectDatabaseByHashImpl(settings, dbInfos); } -void TFederatedWriteSessionImpl::OnFederationStateUpdateImpl() { +std::shared_ptr TFederatedWriteSessionImpl::OnFederationStateUpdateImpl() { Y_ABORT_UNLESS(Lock.IsLocked()); if (!FederationState->Status.IsSuccess()) { // The observer became stale, it won't try to get federation state anymore due to retry policy, // so there's no reason to keep the write session alive. CloseImpl(FederationState->Status.GetStatus(), NYql::TIssues(FederationState->Status.GetIssues())); - return; + return {}; } Y_ABORT_UNLESS(!FederationState->DbInfos.empty()); @@ -271,19 +304,22 @@ void TFederatedWriteSessionImpl::OnFederationStateUpdateImpl() { LOG_LAZY(Log, TLOG_ERR, GetLogPrefixImpl() << message << ". Status: " << status); CloseImpl(status, NYql::TIssues{NYql::TIssue(message)}); } - return; + return {}; } RetryState.reset(); + std::shared_ptr oldSubsession; if (!DatabasesAreSame(preferrableDb, CurrentDatabase)) { LOG_LAZY(Log, TLOG_INFO, GetLogPrefixImpl() << "Start federated write session to database '" << preferrableDb->name() << "' (previous was " << (CurrentDatabase ? CurrentDatabase->name() : "") << ")" << " FederationState: " << *FederationState); - OpenSubsessionImpl(preferrableDb); + oldSubsession = OpenSubsessionImpl(preferrableDb); } ScheduleFederationStateUpdateImpl(UPDATE_FEDERATION_STATE_DELAY); + + return oldSubsession; } void TFederatedWriteSessionImpl::ScheduleFederationStateUpdateImpl(TDuration delay) { @@ -291,8 +327,12 @@ void TFederatedWriteSessionImpl::ScheduleFederationStateUpdateImpl(TDuration del auto cb = [selfCtx = SelfContext](bool ok) { if (ok) { if (auto self = selfCtx->LockShared()) { + std::shared_ptr old; with_lock(self->Lock) { - self->UpdateFederationStateImpl(); + old = self->UpdateFederationStateImpl(); + } + if (old) { + old->Close(TDuration::Zero()); } } } @@ -355,8 +395,6 @@ void TFederatedWriteSessionImpl::WriteEncoded(NTopic::TContinuationToken&& token } void TFederatedWriteSessionImpl::WriteInternal(NTopic::TContinuationToken&&, TWrappedWriteMessage&& wrapped) { - TDeferredWrite deferred(Subsession); - with_lock(Lock) { ClientHasToken = false; if (!wrapped.Message.CreateTimestamp_.has_value()) { @@ -364,15 +402,13 @@ void TFederatedWriteSessionImpl::WriteInternal(NTopic::TContinuationToken&&, TWr } BufferFreeSpace -= wrapped.Message.Data.size(); OriginalMessagesToPassDown.emplace_back(std::move(wrapped)); - PrepareDeferredWriteImpl(deferred); + MaybeWriteImpl(); } - deferred.DoWrite(); - IssueTokenIfAllowed(); } -bool TFederatedWriteSessionImpl::PrepareDeferredWriteImpl(TDeferredWrite& deferred) { +bool TFederatedWriteSessionImpl::MaybeWriteImpl() { Y_ABORT_UNLESS(Lock.IsLocked()); if (!PendingToken.has_value()) { return false; @@ -382,8 +418,7 @@ bool TFederatedWriteSessionImpl::PrepareDeferredWriteImpl(TDeferredWrite& deferr } OriginalMessagesToGetAck.push_back(std::move(OriginalMessagesToPassDown.front())); OriginalMessagesToPassDown.pop_front(); - deferred.Token.emplace(std::move(*PendingToken)); - deferred.Message.emplace(std::move(OriginalMessagesToGetAck.back().Message)); + Subsession->Write(std::move(*PendingToken), std::move(OriginalMessagesToGetAck.back().Message)); PendingToken.reset(); return true; } diff --git a/src/client/federated_topic/impl/federated_write_session.h b/src/client/federated_topic/impl/federated_write_session.h index af8f1b7a70..121eb63fbd 100644 --- a/src/client/federated_topic/impl/federated_write_session.h +++ b/src/client/federated_topic/impl/federated_write_session.h @@ -26,7 +26,8 @@ class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, std::shared_ptr connections, const TFederatedTopicClientSettings& clientSetttings, std::shared_ptr observer, - std::shared_ptr>> codecs); + std::shared_ptr>> codecs, + NTopic::IExecutor::TPtr subsessionHandlersExecutor); ~TFederatedWriteSessionImpl() = default; @@ -61,39 +62,22 @@ class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, } }; - struct TDeferredWrite { - explicit TDeferredWrite(std::shared_ptr writer) - : Writer(std::move(writer)) { - } - - void DoWrite() { - if (!Token.has_value() && !Message.has_value()) { - return; - } - Y_ABORT_UNLESS(Token.has_value() && Message.has_value()); - return Writer->Write(std::move(*Token), std::move(*Message)); - } - - std::shared_ptr Writer; - std::optional Token; - std::optional Message; - }; - private: void Start(); - void OpenSubsessionImpl(std::shared_ptr db); - void OnFederationStateUpdateImpl(); + std::shared_ptr OpenSubsessionImpl(std::shared_ptr db); + std::shared_ptr UpdateFederationStateImpl(); + std::shared_ptr OnFederationStateUpdateImpl(); + void ScheduleFederationStateUpdateImpl(TDuration delay); void WriteInternal(NTopic::TContinuationToken&&, TWrappedWriteMessage&& message); - bool PrepareDeferredWriteImpl(TDeferredWrite& deferred); + bool MaybeWriteImpl(); void CloseImpl(EStatus statusCode, NYql::TIssues&& issues); void CloseImpl(NTopic::TSessionClosedEvent const& ev); bool MessageQueuesAreEmptyImpl() const; - void UpdateFederationStateImpl(); void IssueTokenIfAllowed(); @@ -105,6 +89,7 @@ class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, std::shared_ptr Connections; const NTopic::TTopicClientSettings SubclientSettings; std::shared_ptr>> ProvidedCodecs; + NTopic::IExecutor::TPtr SubsessionHandlersExecutor; NTopic::IRetryPolicy::IRetryState::TPtr RetryState; std::shared_ptr Observer; @@ -121,6 +106,7 @@ class TFederatedWriteSessionImpl : public NTopic::TContinuationTokenIssuer, TAdaptiveLock Lock; + size_t SubsessionGeneration = 0; std::shared_ptr Subsession; std::shared_ptr ClientEventsQueue; @@ -152,8 +138,9 @@ class TFederatedWriteSession : public NTopic::IWriteSession, std::shared_ptr connections, const TFederatedTopicClientSettings& clientSettings, std::shared_ptr observer, - std::shared_ptr>> codecs) - : TContextOwner(settings, std::move(connections), clientSettings, std::move(observer), codecs) {} + std::shared_ptr>> codecs, + NTopic::IExecutor::TPtr subsessionHandlersExecutor) + : TContextOwner(settings, std::move(connections), clientSettings, std::move(observer), codecs, subsessionHandlersExecutor) {} NThreading::TFuture WaitEvent() override { return TryGetImpl()->WaitEvent(); diff --git a/src/client/federated_topic/impl/federation_observer.cpp b/src/client/federated_topic/impl/federation_observer.cpp index 89a9c2d134..53ec045325 100644 --- a/src/client/federated_topic/impl/federation_observer.cpp +++ b/src/client/federated_topic/impl/federation_observer.cpp @@ -147,27 +147,27 @@ void TFederatedDbObserverImpl::OnFederationDiscovery(TStatus&& status, Ydb::Fede FederatedDbState->DbInfos.emplace_back(std::move(db)); } else { - if (!status.IsSuccess()) { + if (status.IsSuccess()) { + ScheduleFederationDiscoveryImpl(REDISCOVERY_DELAY); + } else { + LOG_LAZY(DbDriverState_->Log, TLOG_ERR, TStringBuilder() + << "OnFederationDiscovery: Got error. Status: " << status.GetStatus() + << ". Description: " << status.GetIssues().ToOneLineString()); if (!FederationDiscoveryRetryState) { FederationDiscoveryRetryState = FederationDiscoveryRetryPolicy->CreateRetryState(); } - auto retryDelay = FederationDiscoveryRetryState->GetNextRetryDelay(status.GetStatus()); - - if (retryDelay) { - ScheduleFederationDiscoveryImpl(*retryDelay); + if (auto d = FederationDiscoveryRetryState->GetNextRetryDelay(status.GetStatus())) { + ScheduleFederationDiscoveryImpl(*d); return; } - // If retryDelay is Nothing, meaning there won't be another retry, - // we replace FederatedDbState with the unsuccessful one and then set the PromiseToInitState if needed, - // and the observer becomes stale (see IsStale method). - } else { - ScheduleFederationDiscoveryImpl(REDISCOVERY_DELAY); + // If there won't be another retry, we replace FederatedDbState with the unsuccessful one + // and set the PromiseToInitState to make the observer stale (see IsStale method). } // TODO validate new state and check if differs from previous - auto newInfo = std::make_shared(std::move(result), std::move(status)); + // TODO update only if new state differs std::swap(FederatedDbState, newInfo); } diff --git a/src/client/federated_topic/ut/basic_usage_ut.cpp b/src/client/federated_topic/ut/basic_usage_ut.cpp new file mode 100644 index 0000000000..1a98937b28 --- /dev/null +++ b/src/client/federated_topic/ut/basic_usage_ut.cpp @@ -0,0 +1,1406 @@ +#include +#include + +#include + +#include + +#include +#include +#include + +#include +#include + +#include +#include +#include +#include + +#include + +namespace NYdb::NFederatedTopic::NTests { + +Y_UNIT_TEST_SUITE(BasicUsage) { + + Y_UNIT_TEST(GetAllStartPartitionSessions) { + size_t partitionsCount = 5; + + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2, partitionsCount); + + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr ReadSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); + + // Create read session. + NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; + readSettings + .ConsumerName("shared/user") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(setup->GetTestTopic()); + + ReadSession = topicClient.CreateReadSession(readSettings); + Cerr << "Session was created" << Endl; + + ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); + TMaybe event = ReadSession->GetEvent(false); + Y_ASSERT(!event); + + auto fdsRequest = fdsMock.GetNextPendingRequest(); + Y_ASSERT(fdsRequest.has_value()); + // TODO check fdsRequest->Req db header + + Ydb::FederationDiscovery::ListFederationDatabasesResponse response; + + auto op = response.mutable_operation(); + op->set_status(Ydb::StatusIds::SUCCESS); + response.mutable_operation()->set_ready(true); + response.mutable_operation()->set_id("12345"); + + Ydb::FederationDiscovery::ListFederationDatabasesResult mockResult; + mockResult.set_control_plane_endpoint("cp.logbroker-federation:2135"); + mockResult.set_self_location("fancy_datacenter"); + auto c1 = mockResult.add_federation_databases(); + c1->set_name("dc1"); + c1->set_path("/Root"); + c1->set_id("account-dc1"); + c1->set_endpoint("localhost:" + ToString(fdsMock.Port)); + c1->set_location("dc1"); + c1->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); + c1->set_weight(1000); + auto c2 = mockResult.add_federation_databases(); + c2->set_name("dc2"); + c2->set_path("/Root"); + c2->set_id("account-dc2"); + c2->set_endpoint("localhost:" + ToString(fdsMock.Port)); + c2->set_location("dc2"); + c2->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); + c2->set_weight(500); + + op->mutable_result()->PackFrom(mockResult); + + fdsRequest->Result.SetValue({std::move(response), grpc::Status::OK}); + + for (size_t i = 0; i < partitionsCount; ++i) { + ReadSession->WaitEvent().Wait(); + // Get event + TMaybe event = ReadSession->GetEvent(true/*block - will block if no event received yet*/); + Cerr << "Got new read session event: " << DebugString(*event) << Endl; + + auto* startPartitionSessionEvent = std::get_if(&*event); + Y_ASSERT(startPartitionSessionEvent); + startPartitionSessionEvent->Confirm(); + } + + ReadSession->Close(TDuration::MilliSeconds(10)); + } + + Y_UNIT_TEST(WaitEventBlocksBeforeDiscovery) { + auto setup = std::make_shared(TEST_CASE_NAME, false); + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr ReadSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); + + // Create read session. + NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; + readSettings + .ConsumerName("shared/user") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(setup->GetTestTopic()); + + Cerr << "Before ReadSession was created" << Endl; + ReadSession = topicClient.CreateReadSession(readSettings); + Cerr << "Session was created" << Endl; + + auto f = ReadSession->WaitEvent(); + Cerr << "Returned from WaitEvent" << Endl; + // observer asyncInit should respect client/session timeouts + UNIT_ASSERT(!f.Wait(TDuration::Seconds(1))); + + Cerr << "Session blocked successfully" << Endl; + + UNIT_ASSERT(ReadSession->Close(TDuration::MilliSeconds(10))); + Cerr << "Session closed gracefully" << Endl; + } + + Y_UNIT_TEST(RetryDiscoveryWithCancel) { + // TODO register requests in mock, compare time for retries, reschedules + + auto setup = std::make_shared(TEST_CASE_NAME, false); + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr ReadSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + auto clientSettings = TFederatedTopicClientSettings() + .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( + TDuration::Seconds(10), + TDuration::Seconds(10) + )); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); + + bool answerOk = false; + + for (int i = 0; i < 6; ++i) { + std::optional fdsRequest; + do { + Sleep(TDuration::MilliSeconds(50)); + fdsRequest = fdsMock.GetNextPendingRequest(); + + } while (!fdsRequest.has_value()); + + if (answerOk) { + fdsRequest->Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + } else { + fdsRequest->Result.SetValue({{}, grpc::Status(grpc::StatusCode::UNAVAILABLE, "mock 'unavailable'")}); + } + + answerOk = !answerOk; + } + } + + Y_UNIT_TEST(PropagateSessionClosed) { + auto setup = std::make_shared(TEST_CASE_NAME, false); + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr ReadSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + auto clientSettings = TFederatedTopicClientSettings() + .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( + TDuration::Seconds(10), + TDuration::Seconds(10) + )); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); + + + // Create read session. + NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; + readSettings + .ConsumerName("shared/user") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(setup->GetTestTopic()); + + ReadSession = topicClient.CreateReadSession(readSettings); + Cerr << "Session was created" << Endl; + + Sleep(TDuration::MilliSeconds(50)); + + auto events = ReadSession->GetEvents(false); + UNIT_ASSERT(events.empty()); + + Ydb::FederationDiscovery::ListFederationDatabasesResponse Response; + + auto op = Response.mutable_operation(); + op->set_status(Ydb::StatusIds::SUCCESS); + Response.mutable_operation()->set_ready(true); + Response.mutable_operation()->set_id("12345"); + + Ydb::FederationDiscovery::ListFederationDatabasesResult mockResult; + mockResult.set_control_plane_endpoint("cp.logbroker-federation:2135"); + mockResult.set_self_location("fancy_datacenter"); + auto c1 = mockResult.add_federation_databases(); + c1->set_name("dc1"); + c1->set_path("/Root"); + c1->set_id("account-dc1"); + c1->set_endpoint("localhost:" + ToString(fdsMock.Port)); + c1->set_location("dc1"); + c1->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); + c1->set_weight(1000); + auto c2 = mockResult.add_federation_databases(); + c2->set_name("dc2"); + c2->set_path("/Invalid"); + c2->set_id("account-dc2"); + c2->set_endpoint("localhost:" + ToString(fdsMock.Port)); + c2->set_location("dc2"); + c2->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); + c2->set_weight(500); + + op->mutable_result()->PackFrom(mockResult); + + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue({Response, grpc::Status::OK}); + + NPersQueue::TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + writeSettings.Codec(NPersQueue::ECodec::RAW); + NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + writeSettings.CompressionExecutor(executor); + + auto& client = setup->GetPersQueueClient(); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + TString messageBase = "message----"; + + + ui64 count = 100u; + for (auto i = 0u; i < count; i++) { + auto res = session->Write(messageBase * (200 * 1024) + ToString(i)); + UNIT_ASSERT(res); + + events = ReadSession->GetEvents(true); + UNIT_ASSERT(!events.empty()); + + for (auto& e : events) { + Cerr << ">>> Got event: " << DebugString(e) << Endl; + if (auto* dataEvent = std::get_if(&e)) { + dataEvent->Commit(); + } else if (auto* startPartitionSessionEvent = std::get_if(&e)) { + startPartitionSessionEvent->Confirm(); + } else if (auto* stopPartitionSessionEvent = std::get_if(&e)) { + stopPartitionSessionEvent->Confirm(); + } + } + } + + session->Close(); + } + + Y_UNIT_TEST(RecreateObserver) { + // TODO register requests in mock, compare time for retries, reschedules + + auto setup = std::make_shared(TEST_CASE_NAME, false); + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr ReadSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + auto clientSettings = TFederatedTopicClientSettings() + .RetryPolicy(NTopic::IRetryPolicy::GetNoRetryPolicy()); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); + + + // Create read session. + NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; + readSettings + .ConsumerName("shared/user") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(setup->GetTestTopic()); + + ReadSession = topicClient.CreateReadSession(readSettings); + Cerr << "Session was created" << Endl; + + ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); + auto event = ReadSession->GetEvent(false); + UNIT_ASSERT(!event.Defined()); + + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue({{}, grpc::Status(grpc::StatusCode::UNAVAILABLE, "mock 'unavailable'")}); + + ReadSession->WaitEvent().Wait(); + event = ReadSession->GetEvent(false); + UNIT_ASSERT(event.Defined()); + Cerr << ">>> Got event: " << DebugString(*event) << Endl; + UNIT_ASSERT(std::holds_alternative(*event)); + + auto ReadSession2 = topicClient.CreateReadSession(readSettings); + Cerr << "Session2 was created" << Endl; + + ReadSession2->WaitEvent().Wait(TDuration::Seconds(1)); + event = ReadSession2->GetEvent(false); + UNIT_ASSERT(!event.Defined()); + + fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + event = ReadSession2->GetEvent(true); + UNIT_ASSERT(event.Defined()); + Cerr << ">>> Got event: " << DebugString(*event) << Endl; + UNIT_ASSERT(std::holds_alternative(*event)); + + // Cerr << ">>> Got event: " << DebugString(*event) << Endl; + // UNIT_ASSERT(std::holds_alternative(*event)); + } + + Y_UNIT_TEST(FallbackToSingleDb) { + auto setup = std::make_shared(TEST_CASE_NAME, false); + + setup->Start(true, true); + + std::shared_ptr ReadSession; + + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(setup->GetDriver()); + + // Create read session. + NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; + readSettings + .ConsumerName("shared/user") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(setup->GetTestTopic()); + + ReadSession = topicClient.CreateReadSession(readSettings); + Cerr << "Session was created" << Endl; + + ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); + TMaybe event = ReadSession->GetEvent(false); + Y_ASSERT(event); + Cerr << "Got new read session event: " << DebugString(*event) << Endl; + + auto* startPartitionSessionEvent = std::get_if(&*event); + Y_ASSERT(startPartitionSessionEvent); + startPartitionSessionEvent->Confirm(); + + ReadSession->Close(TDuration::MilliSeconds(10)); + } + + Y_UNIT_TEST(FallbackToSingleDbAfterBadRequest) { + auto setup = std::make_shared(TEST_CASE_NAME, false); + setup->Start(true, true); + TFederationDiscoveryServiceMock fdsMock; + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + fdsMock.Port = setup->GetGrpcPort(); + + Cerr << "PORTS " << fdsMock.Port << " " << newServicePort << Endl; + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + auto clientSettings = TFederatedTopicClientSettings(); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); + + // Create read session. + NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; + readSettings + .ConsumerName("shared/user") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(setup->GetTestTopic()); + + auto ReadSession = topicClient.CreateReadSession(readSettings); + Cerr << "Session was created" << Endl; + + ReadSession->WaitEvent().Wait(TDuration::Seconds(1)); + TMaybe event = ReadSession->GetEvent(false); + Y_ASSERT(!event); + + { + auto fdsRequest = fdsMock.GetNextPendingRequest(); + Y_ASSERT(fdsRequest.has_value()); + Ydb::FederationDiscovery::ListFederationDatabasesResponse response; + auto op = response.mutable_operation(); + op->set_status(Ydb::StatusIds::BAD_REQUEST); + response.mutable_operation()->set_ready(true); + response.mutable_operation()->set_id("12345"); + fdsRequest->Result.SetValue({std::move(response), grpc::Status::OK}); + } + + ReadSession->WaitEvent().Wait(); + TMaybe event2 = ReadSession->GetEvent(true); + Cerr << "Got new read session event: " << DebugString(*event2) << Endl; + + auto* sessionEvent = std::get_if(&*event2); + // At this point the SDK should connect to Topic API, but in this test we have it on a separate port, + // so SDK connects back to FederationDiscovery service and the CLIENT_CALL_UNIMPLEMENTED status is expected. + UNIT_ASSERT_EQUAL(sessionEvent->GetStatus(), EStatus::CLIENT_CALL_UNIMPLEMENTED); + ReadSession->Close(TDuration::MilliSeconds(10)); + } + + Y_UNIT_TEST(SimpleHandlers) { + auto setup = std::make_shared(TEST_CASE_NAME, false); + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr ReadSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + auto clientSettings = TFederatedTopicClientSettings() + .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( + TDuration::Seconds(10), + TDuration::Seconds(10) + )); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); + + ui64 count = 300u; + + TString messageBase = "message----"; + TVector sentMessages; + + for (auto i = 0u; i < count; i++) { + // sentMessages.emplace_back(messageBase * (i+1) + ToString(i)); + sentMessages.emplace_back(messageBase * (10 * i + 1)); + } + + NThreading::TPromise checkedPromise = NThreading::NewPromise(); + auto totalReceived = 0u; + + auto f = checkedPromise.GetFuture(); + TAtomic check = 1; + + // Create read session. + NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; + readSettings + .ConsumerName("shared/user") + .MaxMemoryUsageBytes(1_MB) + .AppendTopics(setup->GetTestTopic()); + + readSettings.FederatedEventHandlers_.SimpleDataHandlers([&](TReadSessionEvent::TDataReceivedEvent& ev) mutable { + Cerr << ">>> event from dataHandler: " << DebugString(ev) << Endl; + Y_VERIFY_S(AtomicGet(check) != 0, "check is false"); + auto& messages = ev.GetMessages(); + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + UNIT_ASSERT_VALUES_EQUAL(message.GetData(), sentMessages[totalReceived]); + totalReceived++; + } + if (totalReceived == sentMessages.size()) + checkedPromise.SetValue(); + }); + + ReadSession = topicClient.CreateReadSession(readSettings); + Cerr << ">>> Session was created" << Endl; + + Sleep(TDuration::MilliSeconds(50)); + + auto events = ReadSession->GetEvents(false); + UNIT_ASSERT(events.empty()); + + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + NPersQueue::TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + writeSettings.Codec(NPersQueue::ECodec::RAW); + NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + writeSettings.CompressionExecutor(executor); + + auto& client = setup->GetPersQueueClient(); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + + for (auto i = 0u; i < count; i++) { + auto res = session->Write(sentMessages[i]); + UNIT_ASSERT(res); + } + + f.GetValueSync(); + ReadSession->Close(); + AtomicSet(check, 0); + } + + Y_UNIT_TEST(ReadMirrored) { + auto setup = std::make_shared(TEST_CASE_NAME, false); + setup->Start(true, true); + setup->CreateTopic(setup->GetTestTopic() + "-mirrored-from-dc2", setup->GetLocalCluster()); + setup->CreateTopic(setup->GetTestTopic() + "-mirrored-from-dc3", setup->GetLocalCluster()); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr ReadSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + auto clientSettings = TFederatedTopicClientSettings() + .RetryPolicy(NTopic::IRetryPolicy::GetFixedIntervalPolicy( + TDuration::Seconds(10), + TDuration::Seconds(10) + )); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); + + ui64 count = 5u; + + TString messageBase = "message----"; + TVector sentMessages; + std::unordered_set sentSet; + + for (auto i = 0u; i < count; i++) { + sentMessages.emplace_back(messageBase * (10 * i + 1)); + sentSet.emplace(sentMessages.back() + "-from-dc1"); + sentSet.emplace(sentMessages.back() + "-from-dc2"); + sentSet.emplace(sentMessages.back() + "-from-dc3"); + } + + NThreading::TPromise checkedPromise = NThreading::NewPromise(); + auto totalReceived = 0u; + + auto f = checkedPromise.GetFuture(); + TAtomic check = 1; + + // Create read session. + NYdb::NFederatedTopic::TFederatedReadSessionSettings readSettings; + readSettings + .ReadMirrored("dc1") + .ConsumerName("shared/user") + .MaxMemoryUsageBytes(16_MB) + .AppendTopics(setup->GetTestTopic()); + + readSettings.FederatedEventHandlers_.SimpleDataHandlers([&](TReadSessionEvent::TDataReceivedEvent& ev) mutable { + Cerr << ">>> event from dataHandler: " << DebugString(ev) << Endl; + Y_VERIFY_S(AtomicGet(check) != 0, "check is false"); + auto& messages = ev.GetMessages(); + Cerr << ">>> get " << messages.size() << " messages in this event" << Endl; + for (size_t i = 0u; i < messages.size(); ++i) { + auto& message = messages[i]; + UNIT_ASSERT(message.GetFederatedPartitionSession()->GetReadSourceDatabaseName() == "dc1"); + UNIT_ASSERT(message.GetFederatedPartitionSession()->GetTopicPath() == setup->GetTestTopic()); + UNIT_ASSERT(message.GetData().EndsWith(message.GetFederatedPartitionSession()->GetTopicOriginDatabaseName())); + + UNIT_ASSERT(!sentSet.empty()); + UNIT_ASSERT_C(sentSet.erase(message.GetData()), "no such element is sentSet: " + message.GetData()); + totalReceived++; + } + if (totalReceived == 3 * sentMessages.size()) { + UNIT_ASSERT(sentSet.empty()); + checkedPromise.SetValue(); + } + }); + + ReadSession = topicClient.CreateReadSession(readSettings); + Cerr << ">>> Session was created" << Endl; + + Sleep(TDuration::MilliSeconds(50)); + + auto events = ReadSession->GetEvents(false); + UNIT_ASSERT(events.empty()); + + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + { + NPersQueue::TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic()).MessageGroupId("src_id"); + writeSettings.Codec(NPersQueue::ECodec::RAW); + NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + writeSettings.CompressionExecutor(executor); + + auto& client = setup->GetPersQueueClient(); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + + for (auto i = 0u; i < count; i++) { + auto res = session->Write(sentMessages[i] + "-from-dc1"); + UNIT_ASSERT(res); + } + + session->Close(); + + Cerr << ">>> Writes to test-topic successful" << Endl; + } + + { + NPersQueue::TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic() + "-mirrored-from-dc2").MessageGroupId("src_id"); + writeSettings.Codec(NPersQueue::ECodec::RAW); + NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + writeSettings.CompressionExecutor(executor); + + auto& client = setup->GetPersQueueClient(); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + + for (auto i = 0u; i < count; i++) { + auto res = session->Write(sentMessages[i] + "-from-dc2"); + UNIT_ASSERT(res); + } + + session->Close(); + + Cerr << ">>> Writes to test-topic-mirrored-from-dc2 successful" << Endl; + } + + { + NPersQueue::TWriteSessionSettings writeSettings; + writeSettings.Path(setup->GetTestTopic() + "-mirrored-from-dc3").MessageGroupId("src_id"); + writeSettings.Codec(NPersQueue::ECodec::RAW); + NPersQueue::IExecutor::TPtr executor = new NTopic::TSyncExecutor(); + writeSettings.CompressionExecutor(executor); + + auto& client = setup->GetPersQueueClient(); + auto session = client.CreateSimpleBlockingWriteSession(writeSettings); + + for (auto i = 0u; i < count; i++) { + auto res = session->Write(sentMessages[i] + "-from-dc3"); + UNIT_ASSERT(res); + } + + session->Close(); + + Cerr << ">>> Writes to test-topic-mirrored-from-dc3 successful" << Endl; + } + + f.GetValueSync(); + ReadSession->Close(); + AtomicSet(check, 0); + } + + Y_UNIT_TEST(BasicWriteSession) { + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); + + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr WriteSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); + + // Create write session. + auto writeSettings = NTopic::TWriteSessionSettings() + .DirectWriteToPartition(false) + .Path(setup->GetTestTopic()) + .MessageGroupId("src_id"); + + WriteSession = topicClient.CreateWriteSession(writeSettings); + Cerr << "Session was created" << Endl; + + WriteSession->WaitEvent().Wait(TDuration::Seconds(1)); + auto event = WriteSession->GetEvent(false); + Y_ASSERT(event); + Cerr << "Got new write session event: " << DebugString(*event) << Endl; + auto* readyToAcceptEvent = std::get_if(&*event); + Y_ASSERT(readyToAcceptEvent); + WriteSession->Write(std::move(readyToAcceptEvent->ContinuationToken), NTopic::TWriteMessage("hello")); + + WriteSession->WaitEvent().Wait(TDuration::Seconds(1)); + event = WriteSession->GetEvent(false); + Y_ASSERT(event); + Cerr << "Got new write session event: " << DebugString(*event) << Endl; + + readyToAcceptEvent = std::get_if(&*event); + Y_ASSERT(readyToAcceptEvent); + + std::optional fdsRequest; + do { + fdsRequest = fdsMock.GetNextPendingRequest(); + if (!fdsRequest.has_value()) { + Sleep(TDuration::MilliSeconds(50)); + } + } while (!fdsRequest.has_value()); + + fdsRequest->Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + WriteSession->WaitEvent().Wait(TDuration::Seconds(1)); + event = WriteSession->GetEvent(false); + Y_ASSERT(event); + Cerr << "Got new write session event: " << DebugString(*event) << Endl; + + auto* acksEvent = std::get_if(&*event); + Y_ASSERT(acksEvent); + + WriteSession->Close(TDuration::MilliSeconds(10)); + } + + Y_UNIT_TEST(CloseWriteSessionImmediately) { + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); + + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + std::shared_ptr WriteSession; + + // Create topic client. + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); + + // Create write session. + auto writeSettings = NTopic::TWriteSessionSettings() + .DirectWriteToPartition(false) + .Path(setup->GetTestTopic()) + .MessageGroupId("src_id"); + + WriteSession = topicClient.CreateWriteSession(writeSettings); + Cerr << "Session was created" << Endl; + + WriteSession->Close(TDuration::MilliSeconds(10)); + } + + Y_UNIT_TEST(WriteSessionCloseWaitsForWrites) { + // Write a bunch of messages before the federation observer initialization. + // Then as soon as the federation discovery service responds to the observer, close the write session. + // The federated write session must wait for acks of all written messages. + + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); + + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); + + // Create write session. + auto writeSettings = NTopic::TWriteSessionSettings() + .DirectWriteToPartition(false) + .Path(setup->GetTestTopic()) + .MessageGroupId("src_id"); + + int acks = 0; + int messageCount = 100; + + auto gotAllAcks = NThreading::NewPromise(); + writeSettings.EventHandlers_.AcksHandler([&](NTopic::TWriteSessionEvent::TAcksEvent& ev) { + acks += ev.Acks.size(); + if (acks == messageCount) { + gotAllAcks.SetValue(); + } + }); + + auto WriteSession = topicClient.CreateWriteSession(writeSettings); + Cerr << "Session was created" << Endl; + + for (int i = 0; i < messageCount; ++i) { + auto event = WriteSession->GetEvent(true); + auto* readyToAcceptEvent = std::get_if(&*event); + WriteSession->Write(std::move(readyToAcceptEvent->ContinuationToken), NTopic::TWriteMessage("hello-" + ToString(i))); + } + + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + // Close method should wait until all messages have been acked. + WriteSession->Close(); + gotAllAcks.GetFuture().Wait(); + UNIT_ASSERT_VALUES_EQUAL(acks, messageCount); + } + + Y_UNIT_TEST(WriteSessionCloseIgnoresWrites) { + // Create a federated topic client with NoRetryPolicy. + // Make federation discovery service to respond with an UNAVAILABLE status. + // It makes a federation observer object to become stale and the write session to close. + + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); + + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + TFederatedTopicClientSettings clientSettings; + clientSettings.RetryPolicy(NPersQueue::IRetryPolicy::GetNoRetryPolicy()); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); + + // Create write session. + auto writeSettings = NTopic::TWriteSessionSettings() + .DirectWriteToPartition(false) + .Path(setup->GetTestTopic()) + .MessageGroupId("src_id"); + + int acks = 0; + writeSettings.EventHandlers_.AcksHandler([&acks](NTopic::TWriteSessionEvent::TAcksEvent& ev) { + acks += ev.Acks.size(); + }); + + auto WriteSession = topicClient.CreateWriteSession(writeSettings); + Cerr << "Session was created" << Endl; + + int messageCount = 100; + for (int i = 0; i < messageCount; ++i) { + auto event = WriteSession->GetEvent(true); + auto* readyToAcceptEvent = std::get_if(&*event); + WriteSession->Write(std::move(readyToAcceptEvent->ContinuationToken), NTopic::TWriteMessage("hello-" + ToString(i))); + } + + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeUnavailableResult()); + + // At this point the observer that federated write session works with should become stale, and the session closes. + // No messages we have written and no federation discovery requests should be sent. + + Sleep(TDuration::Seconds(3)); + UNIT_ASSERT(!fdsMock.GetNextPendingRequest().has_value()); + WriteSession->Close(); + UNIT_ASSERT_VALUES_EQUAL(acks, 0); + } + + Y_UNIT_TEST(PreferredDatabaseNoFallback) { + // The test checks that the session keeps trying to connect to the preferred database + // and does not fall back to other databases. + + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); + + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver); + + auto retryPolicy = std::make_shared(); + + auto writeSettings = TFederatedWriteSessionSettings() + .AllowFallback(false) + .PreferredDatabase("dc2"); + + writeSettings + .RetryPolicy(retryPolicy) + .DirectWriteToPartition(false) + .Path(setup->GetTestTopic()) + .MessageGroupId("src_id"); + + retryPolicy->Initialize(); + retryPolicy->ExpectBreakDown(); + + auto writer = topicClient.CreateWriteSession(writeSettings); + + Ydb::FederationDiscovery::ListFederationDatabasesResponse response; + auto op = response.mutable_operation(); + op->set_status(Ydb::StatusIds::SUCCESS); + response.mutable_operation()->set_ready(true); + response.mutable_operation()->set_id("12345"); + Ydb::FederationDiscovery::ListFederationDatabasesResult mockResult; + mockResult.set_control_plane_endpoint("cp.logbroker-federation:2135"); + mockResult.set_self_location("fancy_datacenter"); + auto c1 = mockResult.add_federation_databases(); + c1->set_name("dc1"); + c1->set_path("/Root"); + c1->set_id("account-dc1"); + c1->set_endpoint("localhost:" + ToString(fdsMock.Port)); + c1->set_location("dc1"); + c1->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); + c1->set_weight(1000); + auto c2 = mockResult.add_federation_databases(); + c2->set_name("dc2"); + c2->set_path("/Root"); + c2->set_id("account-dc2"); + c2->set_endpoint("localhost:" + ToString(fdsMock.Port)); + c2->set_location("dc2"); + c2->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_UNAVAILABLE); + c2->set_weight(500); + op->mutable_result()->PackFrom(mockResult); + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue({std::move(response), grpc::Status::OK}); + + Cerr << "=== Session was created, waiting for retries" << Endl; + retryPolicy->WaitForRetriesSync(3); + + Cerr << "=== In the next federation discovery response dc2 will be available" << Endl; + // fdsMock.PreparedResponse.Clear(); + // std::optional fdsRequest; + fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + Cerr << "=== Waiting for repair" << Endl; + retryPolicy->WaitForRepairSync(); + + Cerr << "=== Closing the session" << Endl; + writer->Close(TDuration::MilliSeconds(10)); + } + + Y_UNIT_TEST(WriteSessionNoAvailableDatabase) { + // Create a federated write session with NoRetryPolicy, PreferredDatabase set and AllowFallback=false. + // Make federation discovery service to respond with an UNAVAILABLE status for the preferred database. + // It makes the write session to close itself. + + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); + + setup->Start(true, true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << newServicePort); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + TFederatedTopicClientSettings clientSettings; + NYdb::NFederatedTopic::TFederatedTopicClient topicClient(driver, clientSettings); + + auto writeSettings = NTopic::TWriteSessionSettings() + .DirectWriteToPartition(false) + .RetryPolicy(NPersQueue::IRetryPolicy::GetNoRetryPolicy()) + .Path(setup->GetTestTopic()) + .MessageGroupId("src_id"); + + auto WriteSession = topicClient.CreateWriteSession(writeSettings); + Cerr << "Session was created" << Endl; + + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultUnavailableDatabases()); + + { + auto e = WriteSession->GetEvent(true); + UNIT_ASSERT(e.Defined()); + Cerr << ">>> Got event: " << DebugString(*e) << Endl; + UNIT_ASSERT(std::holds_alternative(*e)); + } + { + auto e = WriteSession->GetEvent(true); + UNIT_ASSERT(e.Defined()); + Cerr << ">>> Got event: " << DebugString(*e) << Endl; + UNIT_ASSERT(std::holds_alternative(*e)); + } + + WriteSession->Close(); + } + + NTopic::TContinuationToken GetToken(std::shared_ptr writer) { + auto e = writer->GetEvent(true); + UNIT_ASSERT(e.Defined()); + Cerr << ">>> Got event: " << DebugString(*e) << Endl; + auto* readyToAcceptEvent = std::get_if(&*e); + UNIT_ASSERT(readyToAcceptEvent); + return std::move(readyToAcceptEvent->ContinuationToken); + } + + Y_UNIT_TEST(WriteSessionSwitchDatabases) { + // Test that the federated write session doesn't deadlock when reconnecting to another database, + // if the updated state of the federation is different from the previous one. + + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); + + setup->Start(true); + + TFederationDiscoveryServiceMock fdsMock; + fdsMock.Port = setup->GetGrpcPort(); + ui16 newServicePort = setup->GetPortManager()->GetPort(4285); + auto grpcServer = setup->StartGrpcService(newServicePort, &fdsMock); + + auto driverConfig = NYdb::TDriverConfig() + .SetEndpoint(TStringBuilder() << "localhost:" << newServicePort) + .SetDatabase("/Root") + .SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + auto driver = NYdb::TDriver(driverConfig); + auto topicClient = NYdb::NFederatedTopic::TFederatedTopicClient(driver); + + auto writeSettings = NTopic::TFederatedWriteSessionSettings() + .PreferredDatabase("dc1") + .AllowFallback(true) + .DirectWriteToPartition(false) + .RetryPolicy(NPersQueue::IRetryPolicy::GetNoRetryPolicy()) + .Path(setup->GetTestTopic()) + .MessageGroupId("src_id"); + + size_t successfulSessionClosedEvents = 0; + size_t otherSessionClosedEvents = 0; + + writeSettings + .EventHandlers_.SessionClosedHandler([&](const NTopic::TSessionClosedEvent &ev) { + ++(ev.IsSuccess() ? successfulSessionClosedEvents : otherSessionClosedEvents); + }); + + writeSettings.EventHandlers_.HandlersExecutor(NTopic::CreateSyncExecutor()); + + auto WriteSession = topicClient.CreateWriteSession(writeSettings); + + TMaybe token; + + auto fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + { + WriteSession->Write(GetToken(WriteSession), NTopic::TWriteMessage("hello 1")); + token = GetToken(WriteSession); + + auto e = WriteSession->GetEvent(true); + auto* acksEvent = std::get_if(&*e); + UNIT_ASSERT(acksEvent); + } + + // Wait for two requests to the federation discovery service. + // This way we ensure the federated write session has had enough time to request + // the updated state of the federation from its federation observer. + + fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultWithUnavailableDatabase(1)); + + fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultWithUnavailableDatabase(1)); + + { + UNIT_ASSERT(token.Defined()); + WriteSession->Write(std::move(*token), NTopic::TWriteMessage("hello 2")); + + token = GetToken(WriteSession); + + auto e = WriteSession->GetEvent(true); + auto* acksEvent = std::get_if(&*e); + UNIT_ASSERT(acksEvent); + } + + fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + fdsRequest = fdsMock.WaitNextPendingRequest(); + fdsRequest.Result.SetValue(fdsMock.ComposeOkResultAvailableDatabases()); + + { + UNIT_ASSERT(token.Defined()); + WriteSession->Write(std::move(*token), NTopic::TWriteMessage("hello 3")); + + token = GetToken(WriteSession); + + auto e = WriteSession->GetEvent(true); + auto* acksEvent = std::get_if(&*e); + UNIT_ASSERT(acksEvent); + } + + setup->ShutdownGRpc(); + + WriteSession->Close(TDuration::Seconds(5)); + + UNIT_ASSERT_VALUES_EQUAL(otherSessionClosedEvents, 1); + UNIT_ASSERT_VALUES_EQUAL(successfulSessionClosedEvents, 0); + } + + Y_UNIT_TEST(WriteSessionWriteInHandlers) { + // Write messages from all event handlers. It shouldn't deadlock. + + auto setup = std::make_shared( + TEST_CASE_NAME, false, ::NPersQueue::TTestServer::LOGGED_SERVICES, NActors::NLog::PRI_DEBUG, 2); + setup->Start(true, true); + NYdb::TDriverConfig cfg; + cfg.SetEndpoint(TStringBuilder() << "localhost:" << setup->GetGrpcPort()); + cfg.SetDatabase("/Root"); + cfg.SetLog(CreateLogBackend("cerr", ELogPriority::TLOG_DEBUG)); + NYdb::TDriver driver(cfg); + NYdb::NFederatedTopic::TFederatedTopicClient client(driver); + + auto writeSettings = NTopic::TWriteSessionSettings() + .DirectWriteToPartition(false) + .Path(setup->GetTestTopic()) + .MessageGroupId("src_id"); + + std::shared_ptr WriteSession; + + // 1. The session issues the first token: write a message inside AcksHandler. + // 2. The session issues another token: close the session, write a message inside SessionClosedHandler. + + bool gotAcksEvent = false; + std::optional token; + auto [acksHandlerPromise, sessionClosedHandlerPromise] = std::tuple{NThreading::NewPromise(), NThreading::NewPromise()}; + auto [sentFromAcksHandler, sentFromSessionClosedHandler] = std::tuple{acksHandlerPromise.GetFuture(), sessionClosedHandlerPromise.GetFuture()}; + writeSettings.EventHandlers( + NTopic::TWriteSessionSettings::TEventHandlers() + .HandlersExecutor(NTopic::CreateSyncExecutor()) + .ReadyToAcceptHandler([&token](NTopic::TWriteSessionEvent::TReadyToAcceptEvent& e) { + Cerr << "=== Inside ReadyToAcceptHandler" << Endl; + token = std::move(e.ContinuationToken); + }) + .AcksHandler([&token, &gotAcksEvent, &WriteSession, promise = std::move(acksHandlerPromise)](NTopic::TWriteSessionEvent::TAcksEvent&) mutable { + Cerr << "=== Inside AcksHandler" << Endl; + if (!gotAcksEvent) { + gotAcksEvent = true; + WriteSession->Write(std::move(*token), "From AcksHandler"); + promise.SetValue(); + } + }) + .SessionClosedHandler([&token, &WriteSession, promise = std::move(sessionClosedHandlerPromise)](NTopic::TSessionClosedEvent const&) mutable { + Cerr << "=== Inside SessionClosedHandler" << Endl; + WriteSession->Write(std::move(*token), "From SessionClosedHandler"); + promise.SetValue(); + }) + ); + + Cerr << "=== Before CreateWriteSession" << Endl; + WriteSession = client.CreateWriteSession(writeSettings); + Cerr << "=== Session created" << Endl; + + WriteSession->Write(std::move(*token), "After CreateWriteSession"); + + sentFromAcksHandler.Wait(); + Cerr << "=== AcksHandler has written a message, closing the session" << Endl; + + WriteSession->Close(); + sentFromSessionClosedHandler.Wait(); + Cerr << "=== SessionClosedHandler has 'written' a message" << Endl; + } + + void AddDatabase(std::vector>& dbInfos, int id, int weight) { + auto db = std::make_shared(); + db->set_id(ToString(id)); + db->set_name("db" + ToString(id)); + db->set_location("dc" + ToString(id)); + db->set_weight(weight); + db->set_status(Ydb::FederationDiscovery::DatabaseInfo_Status_AVAILABLE); + dbInfos.push_back(db); + } + + void EnableDatabase(std::vector>& dbInfos, int id) { + dbInfos[id - 1]->set_status(Ydb::FederationDiscovery::DatabaseInfo_Status_AVAILABLE); + } + + void DisableDatabase(std::vector>& dbInfos, int id) { + dbInfos[id - 1]->set_status(Ydb::FederationDiscovery::DatabaseInfo_Status_UNAVAILABLE); + } + + Y_UNIT_TEST(SelectDatabaseByHash) { + std::vector> dbInfos; + using Settings = TFederatedWriteSessionSettings; + + { + auto [db, status] = SelectDatabaseByHashImpl(Settings(), dbInfos); + UNIT_ASSERT(!db); + UNIT_ASSERT_EQUAL(status, EStatus::NOT_FOUND); + } + + AddDatabase(dbInfos, 1, 0); + + { + auto [db, status] = SelectDatabaseByHashImpl(Settings(), dbInfos); + UNIT_ASSERT(!db); + UNIT_ASSERT_EQUAL(status, EStatus::NOT_FOUND); + } + + AddDatabase(dbInfos, 2, 100); + + { + auto [db, status] = SelectDatabaseByHashImpl(Settings(), dbInfos); + UNIT_ASSERT(db); + UNIT_ASSERT_EQUAL(db->id(), "2"); + UNIT_ASSERT_EQUAL(status, EStatus::SUCCESS); + } + } + + Y_UNIT_TEST(SelectDatabase) { + std::vector> dbInfos; + for (int i = 1; i < 11; ++i) { + AddDatabase(dbInfos, i, 1000); + } + + using Settings = TFederatedWriteSessionSettings; + + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | set | not found | - | any | NOT_FOUND | + */ + for (bool allow : {false, true}) { + auto settings = Settings().PreferredDatabase("db0").AllowFallback(allow); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc1"); + UNIT_ASSERT(!db); + UNIT_ASSERT_EQUAL(status, EStatus::NOT_FOUND); + } + } + + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | set | available | - | any | preferred | + */ + for (bool allow : {false, true}) { + auto settings = Settings().PreferredDatabase("db8").AllowFallback(allow); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc1"); + UNIT_ASSERT(db); + UNIT_ASSERT_EQUAL(db->id(), "8"); + } + } + + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | set | unavailable | - | false | UNAVAILABLE | + */ + DisableDatabase(dbInfos, 8); + auto settings = Settings().PreferredDatabase("db8").AllowFallback(false); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc1"); + UNIT_ASSERT(!db); + UNIT_ASSERT_EQUAL(status, EStatus::UNAVAILABLE); + EnableDatabase(dbInfos, 8); + } + + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | set | unavailable | - | true | by hash | + */ + DisableDatabase(dbInfos, 8); + auto settings = Settings().PreferredDatabase("db8").AllowFallback(true); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc1"); + UNIT_ASSERT(db); + UNIT_ASSERT_UNEQUAL(db->id(), "8"); + UNIT_ASSERT_EQUAL(status, EStatus::SUCCESS); + EnableDatabase(dbInfos, 8); + } + + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | unset | - | not found | false | NOT_FOUND | + */ + auto settings = Settings().AllowFallback(false); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc0"); + UNIT_ASSERT(!db); + UNIT_ASSERT_EQUAL(status, EStatus::NOT_FOUND); + } + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | unset | - | not found | true | by hash | + */ + auto settings = Settings().AllowFallback(true); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc0"); + UNIT_ASSERT(db); + UNIT_ASSERT_EQUAL(status, EStatus::SUCCESS); + } + + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | unset | - | available | any | local | + */ + for (bool allow : {false, true}) { + auto settings = Settings().AllowFallback(allow); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc1"); + UNIT_ASSERT(db); + UNIT_ASSERT_EQUAL(db->id(), "1"); + } + } + + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | unset | - | unavailable | false | UNAVAILABLE | + */ + DisableDatabase(dbInfos, 1); + auto settings = Settings().AllowFallback(false); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc1"); + UNIT_ASSERT(!db); + UNIT_ASSERT_EQUAL(status, EStatus::UNAVAILABLE); + EnableDatabase(dbInfos, 1); + } + + { + /* + | PreferredDb | Preferred state | Local state | AllowFallback | Return | + |-------------+-----------------+-------------+---------------+-------------| + | unset | - | unavailable | true | by hash | + */ + DisableDatabase(dbInfos, 1); + auto settings = Settings().AllowFallback(true); + auto [db, status] = SelectDatabaseImpl(settings, dbInfos, "dc1"); + UNIT_ASSERT(db); + UNIT_ASSERT_UNEQUAL(db->id(), "1"); + UNIT_ASSERT_EQUAL(status, EStatus::SUCCESS); + EnableDatabase(dbInfos, 1); + } + } + +} + +} diff --git a/src/client/federated_topic/ut/fds_mock.h b/src/client/federated_topic/ut/fds_mock.h new file mode 100644 index 0000000000..365cd0d962 --- /dev/null +++ b/src/client/federated_topic/ut/fds_mock.h @@ -0,0 +1,178 @@ +#pragma once + +#include "util/string/builder.h" +#include +#include + +#include +#include + +namespace NYdb::NFederatedTopic::NTests { + +// ctor gets ---list of response--- tmaybe{response} +// ListFederationDatabases gets 1 element under lock and respond. otherwise +// create 2 queues, for requests and responses +// getrequest() - put request and returns> +// sendresponse() +class TFederationDiscoveryServiceMock: public Ydb::FederationDiscovery::V1::FederationDiscoveryService::Service { +public: + using TRequest = Ydb::FederationDiscovery::ListFederationDatabasesRequest; + using TResponse = Ydb::FederationDiscovery::ListFederationDatabasesResponse; + + struct TGrpcResult { + TResponse Response; + grpc::Status Status; + }; + + struct TManualRequest { + const TRequest* Request; + NThreading::TPromise Result; + }; + +public: + ~TFederationDiscoveryServiceMock() { + while (auto r = GetNextPendingRequest()) { + r->Result.SetValue({}); + } + } + + std::optional GetNextPendingRequest() { + std::optional result; + with_lock(Lock) { + if (!PendingRequests.empty()) { + result = PendingRequests.front(); + PendingRequests.pop_front(); + } + } + return result; + } + + TManualRequest WaitNextPendingRequest() { + do { + auto result = GetNextPendingRequest(); + if (result.has_value()) { + return *result; + } + Sleep(TDuration::MilliSeconds(50)); + } while (true); + } + + virtual grpc::Status ListFederationDatabases(grpc::ServerContext*, + const TRequest* request, + TResponse* response) override { + Y_UNUSED(request); + + auto p = NThreading::NewPromise(); + auto f = p.GetFuture(); + + with_lock(Lock) { + PendingRequests.push_back({request, std::move(p)}); + } + + f.Wait(TDuration::Seconds(35)); + if (f.HasValue()) { + auto result = f.ExtractValueSync(); + Cerr << ">>> Ready to answer: " << (result.Status.ok() ? "ok" : "err") << Endl; + *response = std::move(result.Response); + return result.Status; + } + + return grpc::Status(grpc::StatusCode::UNKNOWN, "No response after timeout"); + } + + TGrpcResult ComposeOkResult(::Ydb::FederationDiscovery::DatabaseInfo::Status status) { + Ydb::FederationDiscovery::ListFederationDatabasesResponse okResponse; + + auto op = okResponse.mutable_operation(); + op->set_status(Ydb::StatusIds::SUCCESS); + okResponse.mutable_operation()->set_ready(true); + okResponse.mutable_operation()->set_id("12345"); + + Ydb::FederationDiscovery::ListFederationDatabasesResult mockResult; + mockResult.set_control_plane_endpoint("cp.logbroker-federation:2135"); + mockResult.set_self_location("fancy_datacenter"); + auto c1 = mockResult.add_federation_databases(); + c1->set_name("dc1"); + c1->set_path("/Root"); + c1->set_id("account-dc1"); + c1->set_endpoint("localhost:" + ToString(Port)); + c1->set_location("dc1"); + c1->set_status(status); + c1->set_weight(1000); + auto c2 = mockResult.add_federation_databases(); + c2->set_name("dc2"); + c2->set_path("/Root"); + c2->set_id("account-dc2"); + c2->set_endpoint("localhost:" + ToString(Port)); + c2->set_location("dc2"); + c2->set_status(status); + c2->set_weight(500); + auto c3 = mockResult.add_federation_databases(); + c3->set_name("dc3"); + c3->set_path("/Root"); + c3->set_id("account-dc3"); + c3->set_endpoint("localhost:" + ToString(Port)); + c3->set_location("dc3"); + c3->set_status(status); + c3->set_weight(500); + + op->mutable_result()->PackFrom(mockResult); + + return {okResponse, grpc::Status::OK}; + } + + TGrpcResult ComposeOkResultAvailableDatabases() { + return ComposeOkResult(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); + } + + TGrpcResult ComposeOkResultUnavailableDatabases() { + return ComposeOkResult(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_UNAVAILABLE); + } + + TGrpcResult ComposeUnavailableResult() { + Ydb::FederationDiscovery::ListFederationDatabasesResponse response; + auto op = response.mutable_operation(); + op->set_status(Ydb::StatusIds::UNAVAILABLE); + response.mutable_operation()->set_ready(true); + response.mutable_operation()->set_id("12345"); + return {response, grpc::Status::OK}; + } + + TGrpcResult ComposeOkResultWithUnavailableDatabase(int unavailableDb) { + Ydb::FederationDiscovery::ListFederationDatabasesResponse okResponse; + + auto op = okResponse.mutable_operation(); + op->set_status(Ydb::StatusIds::SUCCESS); + okResponse.mutable_operation()->set_ready(true); + okResponse.mutable_operation()->set_id("12345"); + + Ydb::FederationDiscovery::ListFederationDatabasesResult mockResult; + mockResult.set_control_plane_endpoint("cp.logbroker-federation:2135"); + mockResult.set_self_location("fancy_datacenter"); + for (int i = 1; i <= 3; ++i) { + auto c1 = mockResult.add_federation_databases(); + c1->set_name(TStringBuilder() << "dc" << i); + c1->set_path("/Root"); + c1->set_id(TStringBuilder() << "account-dc" << i); + c1->set_endpoint("localhost:" + ToString(Port)); + c1->set_location(TStringBuilder() << "dc" << i); + if (i == unavailableDb) { + c1->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_UNAVAILABLE); + } else { + c1->set_status(::Ydb::FederationDiscovery::DatabaseInfo::Status::DatabaseInfo_Status_AVAILABLE); + } + c1->set_weight(i == 0 ? 1000 : 500); + } + + op->mutable_result()->PackFrom(mockResult); + + return {okResponse, grpc::Status::OK}; + } + +public: + ui16 Port; + std::deque PendingRequests; + TAdaptiveLock Lock; +}; + +} // namespace NYdb::NFederatedTopic::NTests diff --git a/src/client/federated_topic/ut/ya.make b/src/client/federated_topic/ut/ya.make new file mode 100644 index 0000000000..8688c7519d --- /dev/null +++ b/src/client/federated_topic/ut/ya.make @@ -0,0 +1,38 @@ +UNITTEST_FOR(ydb/public/sdk/cpp/client/ydb_federated_topic) + +IF (SANITIZER_TYPE == "thread" OR WITH_VALGRIND) + TIMEOUT(1200) + SIZE(LARGE) + TAG(ya:fat) +ELSE() + TIMEOUT(600) + SIZE(MEDIUM) +ENDIF() + +FORK_SUBTESTS() + +PEERDIR( + library/cpp/testing/gmock_in_unittest + ydb/core/testlib/default + ydb/public/lib/json_value + ydb/public/lib/yson_value + ydb/public/sdk/cpp/client/ydb_driver + ydb/public/sdk/cpp/client/ydb_persqueue_public + ydb/public/sdk/cpp/client/ydb_persqueue_public/include + ydb/public/sdk/cpp/client/ydb_persqueue_public/ut/ut_utils + + ydb/public/sdk/cpp/client/ydb_topic/codecs + ydb/public/sdk/cpp/client/ydb_topic + ydb/public/sdk/cpp/client/ydb_topic/impl + + ydb/public/sdk/cpp/client/ydb_federated_topic + ydb/public/sdk/cpp/client/ydb_federated_topic/impl +) + +YQL_LAST_ABI_VERSION() + +SRCS( + basic_usage_ut.cpp +) + +END() diff --git a/src/client/import/import.cpp b/src/client/import/import.cpp index 64f352c844..1db747edd5 100644 --- a/src/client/import/import.cpp +++ b/src/client/import/import.cpp @@ -19,12 +19,6 @@ using namespace Ydb::Import; /// Common namespace { -TInstant ProtoTimestampToInstant(const google::protobuf::Timestamp& timestamp) { - ui64 us = timestamp.seconds() * 1000000; - us += timestamp.nanos() / 1000; - return TInstant::MicroSeconds(us); -} - std::vector ItemsProgressFromProto(const google::protobuf::RepeatedPtrField& proto) { std::vector result; result.reserve(proto.size()); diff --git a/src/client/persqueue_public/impl/persqueue.cpp b/src/client/persqueue_public/impl/persqueue.cpp index 366e54170b..82f5c7332f 100644 --- a/src/client/persqueue_public/impl/persqueue.cpp +++ b/src/client/persqueue_public/impl/persqueue.cpp @@ -85,11 +85,20 @@ TDescribeTopicResult::TDescribeTopicResult(TStatus status, const Ydb::PersQueue: } TDescribeTopicResult::TTopicSettings::TTopicSettings(const Ydb::PersQueue::V1::TopicSettings& settings) { - - PartitionsCount_ = settings.partitions_count(); RetentionPeriod_ = TDuration::MilliSeconds(settings.retention_period_ms()); SupportedFormat_ = static_cast(settings.supported_format()); + if (settings.has_auto_partitioning_settings()) { + PartitionsCount_ = settings.auto_partitioning_settings().min_active_partitions(); + MaxPartitionsCount_ = settings.auto_partitioning_settings().max_active_partitions(); + StabilizationWindow_ = TDuration::Seconds(settings.auto_partitioning_settings().partition_write_speed().stabilization_window().seconds()); + UpUtilizationPercent_ = settings.auto_partitioning_settings().partition_write_speed().up_utilization_percent(); + DownUtilizationPercent_ = settings.auto_partitioning_settings().partition_write_speed().down_utilization_percent(); + AutoPartitioningStrategy_ = settings.auto_partitioning_settings().strategy(); + } else { + PartitionsCount_ = settings.partitions_count(); + } + for (const auto& codec : settings.supported_codecs()) { SupportedCodecs_.push_back(static_cast(codec)); } diff --git a/src/client/persqueue_public/impl/persqueue_impl.h b/src/client/persqueue_public/impl/persqueue_impl.h index a0edce9ca0..a653e3c328 100644 --- a/src/client/persqueue_public/impl/persqueue_impl.h +++ b/src/client/persqueue_public/impl/persqueue_impl.h @@ -55,6 +55,34 @@ class TPersQueueClient::TImpl : public TClientImplCommonset_max_active_partitions(settings.PartitionsCount_); + autoPartitioningSettingsDefined = true; + } + if (settings.AutoPartitioningStrategy_.has_value()) { + props.mutable_auto_partitioning_settings()->set_strategy(*settings.AutoPartitioningStrategy_); + autoPartitioningSettingsDefined = true; + } + if (settings.DownUtilizationPercent_.has_value()) { + props.mutable_auto_partitioning_settings()->mutable_partition_write_speed()->set_down_utilization_percent(*settings.DownUtilizationPercent_); + autoPartitioningSettingsDefined = true; + } + if (settings.UpUtilizationPercent_.has_value()) { + props.mutable_auto_partitioning_settings()->mutable_partition_write_speed()->set_up_utilization_percent(*settings.UpUtilizationPercent_); + autoPartitioningSettingsDefined = true; + } + if (settings.StabilizationWindow_.has_value()) { + props.mutable_auto_partitioning_settings()->mutable_partition_write_speed()->mutable_stabilization_window()->set_seconds((*settings.StabilizationWindow_).Seconds()); + autoPartitioningSettingsDefined = true; + } + + if (!autoPartitioningSettingsDefined) { + props.set_partitions_count(settings.PartitionsCount_); + } else { + props.mutable_auto_partitioning_settings()->set_min_active_partitions(settings.PartitionsCount_); + } + props.set_retention_period_ms(settings.RetentionPeriod_.MilliSeconds()); props.set_supported_format(static_cast(settings.SupportedFormat_)); for (const auto& codec : settings.SupportedCodecs_) { diff --git a/src/client/persqueue_public/impl/write_session_impl.cpp b/src/client/persqueue_public/impl/write_session_impl.cpp index 54ac517ca0..67d5e95143 100644 --- a/src/client/persqueue_public/impl/write_session_impl.cpp +++ b/src/client/persqueue_public/impl/write_session_impl.cpp @@ -93,10 +93,7 @@ TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStat LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session is aborting and will not restart"); return result; } - LOG_LAZY(DbDriverState->Log, TLOG_ERR, - LogPrefix() << "Got error. Status: " << status.Status - << ". Description: " << IssuesSingleLineString(status.Issues) - ); + SessionEstablished = false; if (!RetryState) { RetryState = Settings.RetryPolicy_->CreateRetryState(); @@ -106,13 +103,11 @@ TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStat if (nextDelay) { result.StartDelay = *nextDelay; result.DoRestart = true; - LOG_LAZY(DbDriverState->Log, - TLOG_DEBUG, - LogPrefix() << "Write session will restart in " << result.StartDelay.MilliSeconds() << " ms" - ); + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Got error. " << status.ToDebugString()); + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will restart in " << result.StartDelay); ResetForRetryImpl(); - } else { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Got error. " << status.ToDebugString()); LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Write session will not restart after a fatal error"); result.DoStop = true; CheckHandleResultImpl(result); diff --git a/src/client/persqueue_public/include/control_plane.h b/src/client/persqueue_public/include/control_plane.h index c8cc2ec7e3..0511fb8e50 100644 --- a/src/client/persqueue_public/include/control_plane.h +++ b/src/client/persqueue_public/include/control_plane.h @@ -117,6 +117,12 @@ struct TDescribeTopicResult : public TStatus { } GETTER(std::optional, RemoteMirrorRule); + GETTER(std::optional, MaxPartitionsCount); + GETTER(std::optional, StabilizationWindow); + GETTER(std::optional, UpUtilizationPercent); + GETTER(std::optional, DownUtilizationPercent); + GETTER(std::optional, AutoPartitioningStrategy); + #undef GETTER @@ -138,6 +144,12 @@ struct TDescribeTopicResult : public TStatus { std::optional AbcId_; std::optional AbcSlug_; std::string FederationAccount_; + + std::optional MaxPartitionsCount_; + std::optional StabilizationWindow_; + std::optional UpUtilizationPercent_; + std::optional DownUtilizationPercent_; + std::optional AutoPartitioningStrategy_; }; TDescribeTopicResult(TStatus status, const Ydb::PersQueue::V1::DescribeTopicResult& result); @@ -191,6 +203,7 @@ struct TReadRuleSettings { // Settings for topic. template struct TTopicSettings : public TOperationRequestSettings { + friend class TPersQueueClient; struct TRemoteMirrorRuleSettings { TRemoteMirrorRuleSettings() {} @@ -266,9 +279,22 @@ struct TTopicSettings : public TOperationRequestSettings { if (settings.RemoteMirrorRule()) { RemoteMirrorRule_ = TRemoteMirrorRuleSettings().SetSettings(settings.RemoteMirrorRule().value()); } + + MaxPartitionsCount_ = settings.MaxPartitionsCount(); + StabilizationWindow_ = settings.StabilizationWindow(); + UpUtilizationPercent_ = settings.UpUtilizationPercent(); + DownUtilizationPercent_ = settings.DownUtilizationPercent(); + AutoPartitioningStrategy_ = settings.AutoPartitioningStrategy(); + return static_cast(*this); } +private: + std::optional MaxPartitionsCount_; + std::optional StabilizationWindow_; + std::optional UpUtilizationPercent_; + std::optional DownUtilizationPercent_; + std::optional AutoPartitioningStrategy_; }; diff --git a/src/client/query/client.cpp b/src/client/query/client.cpp index 5c3b842964..6f3e6636f4 100644 --- a/src/client/query/client.cpp +++ b/src/client/query/client.cpp @@ -21,7 +21,8 @@ namespace NYdb::NQuery { -using TRetryContextAsync = NRetry::Async::TRetryContext; +using TRetryContextResultAsync = NRetry::Async::TRetryContext; +using TRetryContextAsync = NRetry::Async::TRetryContext; NYdb::NRetry::TRetryOperationSettings GetRetrySettings(TDuration timeout, bool isIndempotent) { return NYdb::NRetry::TRetryOperationSettings() @@ -89,6 +90,7 @@ class TQueryClient::TImpl: public TClientImplCommon, public auto request = MakeOperationRequest(settings); request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); + request.set_pool_id(settings.PoolId_); request.mutable_script_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); request.mutable_script_content()->set_text(TStringType{script}); SetDuration(settings.ResultsTtl_, *request.mutable_results_ttl()); @@ -576,12 +578,32 @@ int64_t TQueryClient::GetCurrentPoolSize() const { return Impl_->GetCurrentPoolSize(); } -TAsyncExecuteQueryResult TQueryClient::RetryQuery(TQueryFunc&& queryFunc, TRetryOperationSettings settings) +TAsyncExecuteQueryResult TQueryClient::RetryQuery(TQueryResultFunc&& queryFunc, TRetryOperationSettings settings) { + TRetryContextResultAsync::TPtr ctx(new NRetry::Async::TRetryWithSession(*this, std::move(queryFunc), settings)); + return ctx->Execute(); +} + +TAsyncStatus TQueryClient::RetryQuery(TQueryFunc&& queryFunc, TRetryOperationSettings settings) { TRetryContextAsync::TPtr ctx(new NRetry::Async::TRetryWithSession(*this, std::move(queryFunc), settings)); return ctx->Execute(); } +TAsyncStatus TQueryClient::RetryQuery(TQueryWithoutSessionFunc&& queryFunc, TRetryOperationSettings settings) { + TRetryContextAsync::TPtr ctx(new NRetry::Async::TRetryWithoutSession(*this, std::move(queryFunc), settings)); + return ctx->Execute(); +} + +TStatus TQueryClient::RetryQuery(const TQuerySyncFunc& queryFunc, TRetryOperationSettings settings) { + NRetry::Sync::TRetryWithSession ctx(*this, queryFunc, settings); + return ctx.Execute(); +} + +TStatus TQueryClient::RetryQuery(const TQueryWithoutSessionSyncFunc& queryFunc, TRetryOperationSettings settings) { + NRetry::Sync::TRetryWithoutSession ctx(*this, queryFunc, settings); + return ctx.Execute(); +} + TAsyncExecuteQueryResult TQueryClient::RetryQuery(const std::string& query, const TTxControl& txControl, TDuration timeout, bool isIndempotent) { @@ -589,7 +611,7 @@ TAsyncExecuteQueryResult TQueryClient::RetryQuery(const std::string& query, cons auto queryFunc = [&query, &txControl](TSession session, TDuration duration) -> TAsyncExecuteQueryResult { return session.ExecuteQuery(query, txControl, TExecuteQuerySettings().ClientTimeout(duration)); }; - TRetryContextAsync::TPtr ctx(new NRetry::Async::TRetryWithSession(*this, std::move(queryFunc), settings)); + TRetryContextResultAsync::TPtr ctx(new NRetry::Async::TRetryWithSession(*this, std::move(queryFunc), settings)); return ctx->Execute(); } diff --git a/src/client/query/impl/exec_query.cpp b/src/client/query/impl/exec_query.cpp index 63dcf24a3c..a570cb72ea 100644 --- a/src/client/query/impl/exec_query.cpp +++ b/src/client/query/impl/exec_query.cpp @@ -212,6 +212,7 @@ TFuture> StreamExecuteQueryIm auto request = MakeRequest(); request.set_exec_mode(::Ydb::Query::ExecMode(settings.ExecMode_)); request.set_stats_mode(::Ydb::Query::StatsMode(settings.StatsMode_)); + request.set_pool_id(settings.PoolId_); request.mutable_query_content()->set_text(TStringType{query}); request.mutable_query_content()->set_syntax(::Ydb::Query::Syntax(settings.Syntax_)); if (session.has_value()) { diff --git a/src/client/scheme/scheme.cpp b/src/client/scheme/scheme.cpp index bffe90cc0a..b37beebce6 100644 --- a/src/client/scheme/scheme.cpp +++ b/src/client/scheme/scheme.cpp @@ -17,6 +17,13 @@ namespace NScheme { using namespace NThreading; using namespace Ydb::Scheme; +void TPermissions::SerializeTo(::Ydb::Scheme::Permissions& proto) const { + proto.set_subject(Subject); + for (const auto& name : PermissionNames) { + *proto.mutable_permission_names()->Add() = name; + } +} + TVirtualTimestamp::TVirtualTimestamp(uint64_t planStep, uint64_t txId) : PlanStep(planStep) , TxId(txId) @@ -95,6 +102,8 @@ static ESchemeEntryType ConvertProtoEntryType(::Ydb::Scheme::Entry::Type entry) return ESchemeEntryType::ExternalDataSource; case ::Ydb::Scheme::Entry::VIEW: return ESchemeEntryType::View; + case ::Ydb::Scheme::Entry::RESOURCE_POOL: + return ESchemeEntryType::ResourcePool; default: return ESchemeEntryType::Unknown; } @@ -120,6 +129,13 @@ void TSchemeEntry::Out(IOutputStream& out) const { << " }"; } +void TSchemeEntry::SerializeTo(::Ydb::Scheme::ModifyPermissionsRequest& request) const { + request.mutable_actions()->Add()->set_change_owner(Owner); + for (const auto& permission : Permissions) { + permission.SerializeTo(*request.mutable_actions()->Add()->mutable_set()); + } +} + class TSchemeClient::TImpl : public TClientImplCommon { public: TImpl(std::shared_ptr&& connections, const TCommonClientSettings& settings) diff --git a/src/client/table/out.cpp b/src/client/table/out.cpp index 2f24688a79..401695350c 100644 --- a/src/client/table/out.cpp +++ b/src/client/table/out.cpp @@ -23,3 +23,69 @@ Y_DECLARE_OUT_SPEC(, NYdb::NTable::TCreateSessionResult, o, x) { Y_DECLARE_OUT_SPEC(, NYdb::NTable::TDescribeTableResult, o, x) { return x.Out(o); } + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings::EDistance, stream, value) { + auto convertDistance = [] (auto value) -> auto { + switch (value) { + case NYdb::NTable::TVectorIndexSettings::EDistance::Cosine: + return "COSINE"; + case NYdb::NTable::TVectorIndexSettings::EDistance::Manhattan: + return "MANHATTAN"; + case NYdb::NTable::TVectorIndexSettings::EDistance::Euclidean: + return "EUCLIDEAN"; + case NYdb::NTable::TVectorIndexSettings::EDistance::Unknown: + return "UNKNOWN"; + } + }; + + stream << convertDistance(value); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings::ESimilarity, stream, value) { + auto convertSimilarity = [] (auto value) -> auto { + switch (value) { + case NYdb::NTable::TVectorIndexSettings::ESimilarity::Cosine: + return "COSINE"; + case NYdb::NTable::TVectorIndexSettings::ESimilarity::InnerProduct: + return "INNER_PRODUCT"; + case NYdb::NTable::TVectorIndexSettings::ESimilarity::Unknown: + return "UNKNOWN"; + } + }; + + stream << convertSimilarity(value); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings::EVectorType, stream, value) { + auto convertVectorType = [] (auto value) -> auto { + switch (value) { + case NYdb::NTable::TVectorIndexSettings::EVectorType::Float: + return "FLOAT"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Uint8: + return "UINT8"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Int8: + return "INT8"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Bit: + return "BIT"; + case NYdb::NTable::TVectorIndexSettings::EVectorType::Unknown: + return "UNKNOWN"; + } + }; + + stream << convertVectorType(value); +} + +Y_DECLARE_OUT_SPEC(, NYdb::NTable::TVectorIndexSettings, stream, value) { + stream << "{"; + + if (const auto* distance = std::get_if(&value.Metric)) { + stream << " distance: " << *distance << ""; + } else if (const auto* similarity = std::get_if(&value.Metric)) { + stream << " similarity: " << *similarity << ""; + } + + stream << ", vector_type: " << value.VectorType << ""; + stream << ", vector_dimension: " << value.VectorDimension << ""; + + stream << " }"; +} diff --git a/src/client/table/table.cpp b/src/client/table/table.cpp index ec6d575c2e..67df1cf1cc 100644 --- a/src/client/table/table.cpp +++ b/src/client/table/table.cpp @@ -27,6 +27,7 @@ #include #include +#include #include @@ -251,6 +252,24 @@ static void SerializeTo(const TRenameIndex& rename, Ydb::Table::RenameIndexItem& proto.set_replace_destination(rename.ReplaceDestination_); } +template +TExplicitPartitions TExplicitPartitions::FromProto(const TProto& proto) { + TExplicitPartitions out; + for (const auto& splitPoint : proto.split_points()) { + TValue value(TType(splitPoint.type()), splitPoint.value()); + out.AppendSplitPoints(value); + } + return out; +} + +void TExplicitPartitions::SerializeTo(Ydb::Table::ExplicitPartitions& proto) const { + for (const auto& splitPoint : SplitPoints_) { + auto* boundary = proto.add_split_points(); + boundary->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType())); + boundary->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint)); + } +} + class TTableDescription::TImpl { using EUnit = TValueSinceUnixEpochModeSettings::EUnit; @@ -419,13 +438,7 @@ class TTableDescription::TImpl { break; case Ydb::Table::CreateTableRequest::kPartitionAtKeys: { - TExplicitPartitions partitionAtKeys; - for (const auto& splitPoint : request.partition_at_keys().split_points()) { - TValue value(TType(splitPoint.type()), splitPoint.value()); - partitionAtKeys.AppendSplitPoints(value); - } - - SetPartitionAtKeys(partitionAtKeys); + SetPartitionAtKeys(TExplicitPartitions::FromProto(request.partition_at_keys())); break; } @@ -456,6 +469,18 @@ class TTableDescription::TImpl { Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns)); } + void AddSecondaryIndex(const TIndexDescription& indexDescription) { + Indexes_.emplace_back(indexDescription); + } + + void AddVectorIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const TVectorIndexSettings& vectorIndexSettings) { + Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, {}, {}, vectorIndexSettings)); + } + + void AddVectorIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns, const TVectorIndexSettings& vectorIndexSettings) { + Indexes_.emplace_back(TIndexDescription(indexName, type, indexColumns, dataColumns, {}, vectorIndexSettings)); + } + void AddChangefeed(const std::string& name, EChangefeedMode mode, EChangefeedFormat format) { Changefeeds_.emplace_back(name, mode, format); } @@ -729,6 +754,10 @@ void TTableDescription::AddSecondaryIndex(const std::string& indexName, EIndexTy Impl_->AddSecondaryIndex(indexName, type, indexColumns, dataColumns); } +void TTableDescription::AddSecondaryIndex(const TIndexDescription& indexDescription) { + Impl_->AddSecondaryIndex(indexDescription); +} + void TTableDescription::AddSyncSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { AddSecondaryIndex(indexName, EIndexType::GlobalSync, indexColumns); } @@ -753,6 +782,14 @@ void TTableDescription::AddUniqueSecondaryIndex(const std::string& indexName, co AddSecondaryIndex(indexName, EIndexType::GlobalUnique, indexColumns, dataColumns); } +void TTableDescription::AddVectorKMeansTreeSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const TVectorIndexSettings& vectorIndexSettings) { + Impl_->AddVectorIndex(indexName, EIndexType::GlobalVectorKMeansTree, indexColumns, vectorIndexSettings); +} + +void TTableDescription::AddVectorKMeansTreeSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TVectorIndexSettings& vectorIndexSettings) { + Impl_->AddVectorIndex(indexName, EIndexType::GlobalVectorKMeansTree, indexColumns, dataColumns, vectorIndexSettings); +} + void TTableDescription::AddSecondaryIndex(const std::string& indexName, const std::vector& indexColumns) { AddSyncSecondaryIndex(indexName, indexColumns); } @@ -922,12 +959,7 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con } if (const auto& partitionAtKeys = Impl_->GetPartitionAtKeys()) { - auto* borders = request.mutable_partition_at_keys(); - for (const auto& splitPoint : partitionAtKeys->SplitPoints_) { - auto* border = borders->add_split_points(); - border->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType())); - border->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint)); - } + partitionAtKeys->SerializeTo(*request.mutable_partition_at_keys()); } else if (Impl_->GetProto().shard_key_bounds_size()) { request.mutable_partition_at_keys()->mutable_split_points()->CopyFrom(Impl_->GetProto().shard_key_bounds()); } @@ -1150,6 +1182,11 @@ TTableBuilder& TTableBuilder::SetPrimaryKeyColumn(const std::string& primaryKeyC return *this; } +TTableBuilder& TTableBuilder::AddSecondaryIndex(const TIndexDescription& indexDescription) { + TableDescription_.AddSecondaryIndex(indexDescription); + return *this; +} + TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, EIndexType type, const std::vector& indexColumns, const std::vector& dataColumns) { TableDescription_.AddSecondaryIndex(indexName, type, indexColumns, dataColumns); return *this; @@ -1205,6 +1242,16 @@ TTableBuilder& TTableBuilder::AddUniqueSecondaryIndex(const std::string& indexNa return AddSecondaryIndex(indexName, EIndexType::GlobalUnique, indexColumns); } +TTableBuilder& TTableBuilder::AddVectorKMeansTreeSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const std::vector& dataColumns, const TVectorIndexSettings& vectorIndexSettings) { + TableDescription_.AddVectorKMeansTreeSecondaryIndex(indexName, indexColumns, dataColumns, vectorIndexSettings); + return *this; +} + +TTableBuilder& TTableBuilder::AddVectorKMeansTreeSecondaryIndex(const std::string& indexName, const std::vector& indexColumns, const TVectorIndexSettings& vectorIndexSettings) { + TableDescription_.AddVectorKMeansTreeSecondaryIndex(indexName, indexColumns, vectorIndexSettings); + return *this; +} + TTableBuilder& TTableBuilder::AddSecondaryIndex(const std::string& indexName, const std::string& indexColumn) { return AddSyncSecondaryIndex(indexName, indexColumn); } @@ -2204,16 +2251,27 @@ bool TRenameItem::ReplaceDestination() const { //////////////////////////////////////////////////////////////////////////////// -TIndexDescription::TIndexDescription(const std::string& name, EIndexType type, - const std::vector& indexColumns, const std::vector& dataColumns) - : IndexName_(name) +TIndexDescription::TIndexDescription( + const std::string& name, + EIndexType type, + const std::vector& indexColumns, + const std::vector& dataColumns, + const std::vector& globalIndexSettings, + const std::optional& vectorIndexSettings +) : IndexName_(name) , IndexType_(type) , IndexColumns_(indexColumns) , DataColumns_(dataColumns) + , GlobalIndexSettings_(globalIndexSettings) + , VectorIndexSettings_(vectorIndexSettings) {} -TIndexDescription::TIndexDescription(const std::string& name, const std::vector& indexColumns, const std::vector& dataColumns) - : TIndexDescription(name, EIndexType::GlobalSync, indexColumns, dataColumns) +TIndexDescription::TIndexDescription( + const std::string& name, + const std::vector& indexColumns, + const std::vector& dataColumns, + const std::vector& globalIndexSettings +) : TIndexDescription(name, EIndexType::GlobalSync, indexColumns, dataColumns, globalIndexSettings) {} TIndexDescription::TIndexDescription(const Ydb::Table::TableIndex& tableIndex) @@ -2240,15 +2298,169 @@ const std::vector& TIndexDescription::GetDataColumns() const { return DataColumns_; } +const std::optional& TIndexDescription::GetVectorIndexSettings() const { + return VectorIndexSettings_; +} + uint64_t TIndexDescription::GetSizeBytes() const { return SizeBytes; } +template +TGlobalIndexSettings TGlobalIndexSettings::FromProto(const TProto& proto) { + auto partitionsFromProto = [](const auto& proto) -> TUniformOrExplicitPartitions { + switch (proto.partitions_case()) { + case TProto::kUniformPartitions: + return proto.uniform_partitions(); + case TProto::kPartitionAtKeys: + return TExplicitPartitions::FromProto(proto.partition_at_keys()); + default: + return {}; + } + }; + + return { + .PartitioningSettings = TPartitioningSettings(proto.partitioning_settings()), + .Partitions = partitionsFromProto(proto) + }; +} + +void TGlobalIndexSettings::SerializeTo(Ydb::Table::GlobalIndexSettings& settings) const { + *settings.mutable_partitioning_settings() = PartitioningSettings.GetProto(); + + auto variantVisitor = [&settings](auto&& partitions) { + using T = std::decay_t; + if constexpr (std::is_same_v) { + settings.set_uniform_partitions(partitions); + } else if constexpr (std::is_same_v) { + partitions.SerializeTo(*settings.mutable_partition_at_keys()); + } + }; + std::visit(std::move(variantVisitor), Partitions); +} + +template +TVectorIndexSettings TVectorIndexSettings::FromProto(const TProto& proto) { + auto convertDistance = [] (auto distance) -> auto { + switch (distance) { + case Ydb::Table::VectorIndexSettings::DISTANCE_COSINE: + return EDistance::Cosine; + case Ydb::Table::VectorIndexSettings::DISTANCE_MANHATTAN: + return EDistance::Manhattan; + case Ydb::Table::VectorIndexSettings::DISTANCE_EUCLIDEAN: + return EDistance::Euclidean; + default: + return EDistance::Unknown; + } + }; + + auto convertSimilarity = [] (auto similarity) -> auto { + switch (similarity) { + case Ydb::Table::VectorIndexSettings::SIMILARITY_COSINE: + return ESimilarity::Cosine; + case Ydb::Table::VectorIndexSettings::SIMILARITY_INNER_PRODUCT: + return ESimilarity::InnerProduct; + default: + return ESimilarity::Unknown; + } + }; + + auto convertVectorType = [] (auto vectorType) -> auto { + switch (vectorType) { + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT: + return EVectorType::Float; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT8: + return EVectorType::Uint8; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT8: + return EVectorType::Int8; + case Ydb::Table::VectorIndexSettings::VECTOR_TYPE_BIT: + return EVectorType::Bit; + default: + return EVectorType::Unknown; + } + }; + + + auto metricFromProto = [&](const auto& proto) -> TVectorIndexSettings::TMetric { + switch (proto.metric_case()) { + case TProto::kDistance: + return convertDistance(proto.distance()); + case TProto::kSimilarity: + return convertSimilarity(proto.similarity()); + default: + return {}; + } + }; + + return { + .Metric = metricFromProto(proto), + .VectorType = convertVectorType(proto.vector_type()), + .VectorDimension = proto.vector_dimension() + }; +} + +void TVectorIndexSettings::SerializeTo(Ydb::Table::VectorIndexSettings& settings) const { + auto convertDistance = [] (auto distance) -> auto { + switch (distance) { + case EDistance::Cosine: + return Ydb::Table::VectorIndexSettings::DISTANCE_COSINE; + case EDistance::Manhattan: + return Ydb::Table::VectorIndexSettings::DISTANCE_MANHATTAN; + case EDistance::Euclidean: + return Ydb::Table::VectorIndexSettings::DISTANCE_EUCLIDEAN; + case EDistance::Unknown: + return Ydb::Table::VectorIndexSettings::DISTANCE_UNSPECIFIED; + } + }; + + auto convertSimilarity = [] (auto similarity) -> auto { + switch (similarity) { + case ESimilarity::Cosine: + return Ydb::Table::VectorIndexSettings::SIMILARITY_COSINE; + case ESimilarity::InnerProduct: + return Ydb::Table::VectorIndexSettings::SIMILARITY_INNER_PRODUCT; + case ESimilarity::Unknown: + return Ydb::Table::VectorIndexSettings::SIMILARITY_UNSPECIFIED; + } + }; + + auto convertVectorType = [] (auto vectorType) -> auto { + switch (vectorType) { + case EVectorType::Float: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_FLOAT; + case EVectorType::Uint8: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UINT8; + case EVectorType::Int8: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_INT8; + case EVectorType::Bit: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_BIT; + case EVectorType::Unknown: + return Ydb::Table::VectorIndexSettings::VECTOR_TYPE_UNSPECIFIED; + } + }; + + + if (const auto* distance = std::get_if(&Metric)) { + settings.set_distance(convertDistance(*distance)); + } else if (const auto* similarity = std::get_if(&Metric)) { + settings.set_similarity(convertSimilarity(*similarity)); + } + + settings.set_vector_type(convertVectorType(VectorType)); + settings.set_vector_dimension(VectorDimension); +} + +void TVectorIndexSettings::Out(IOutputStream& o) const { + o << *this; +} + template TIndexDescription TIndexDescription::FromProto(const TProto& proto) { EIndexType type; std::vector indexColumns; std::vector dataColumns; + std::vector globalIndexSettings; + std::optional vectorIndexSettings; indexColumns.assign(proto.index_columns().begin(), proto.index_columns().end()); dataColumns.assign(proto.data_columns().begin(), proto.data_columns().end()); @@ -2256,19 +2468,31 @@ TIndexDescription TIndexDescription::FromProto(const TProto& proto) { switch (proto.type_case()) { case TProto::kGlobalIndex: type = EIndexType::GlobalSync; + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(proto.global_index().settings())); break; case TProto::kGlobalAsyncIndex: type = EIndexType::GlobalAsync; + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(proto.global_async_index().settings())); break; case TProto::kGlobalUniqueIndex: type = EIndexType::GlobalUnique; + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(proto.global_unique_index().settings())); + break; + case TProto::kGlobalVectorKmeansTreeIndex: { + type = EIndexType::GlobalVectorKMeansTree; + const auto &vectorProto = proto.global_vector_kmeans_tree_index(); + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(vectorProto.level_table_settings())); + globalIndexSettings.emplace_back(TGlobalIndexSettings::FromProto(vectorProto.posting_table_settings())); + vectorIndexSettings = TVectorIndexSettings::FromProto(vectorProto.vector_settings()); break; + } default: // fallback to global sync type = EIndexType::GlobalSync; + globalIndexSettings.resize(1); break; } - auto result = TIndexDescription(proto.name(), type, indexColumns, dataColumns); + auto result = TIndexDescription(proto.name(), type, indexColumns, dataColumns, globalIndexSettings, vectorIndexSettings); if constexpr (std::is_same_v) { result.SizeBytes = proto.size_bytes(); } @@ -2285,15 +2509,38 @@ void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const { *proto.mutable_data_columns() = {DataColumns_.begin(), DataColumns_.end()}; switch (IndexType_) { - case EIndexType::GlobalSync: - *proto.mutable_global_index() = Ydb::Table::GlobalIndex(); + case EIndexType::GlobalSync: { + auto& settings = *proto.mutable_global_index()->mutable_settings(); + if (GlobalIndexSettings_.size() == 1) + GlobalIndexSettings_[0].SerializeTo(settings); + break; + } + case EIndexType::GlobalAsync: { + auto& settings = *proto.mutable_global_async_index()->mutable_settings(); + if (GlobalIndexSettings_.size() == 1) + GlobalIndexSettings_[0].SerializeTo(settings); break; - case EIndexType::GlobalAsync: - *proto.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex(); + } + case EIndexType::GlobalUnique: { + auto& settings = *proto.mutable_global_unique_index()->mutable_settings(); + if (GlobalIndexSettings_.size() == 1) + GlobalIndexSettings_[0].SerializeTo(settings); break; - case EIndexType::GlobalUnique: - *proto.mutable_global_unique_index() = Ydb::Table::GlobalUniqueIndex(); + } + case EIndexType::GlobalVectorKMeansTree: { + auto* global_vector_kmeans_tree_index = proto.mutable_global_vector_kmeans_tree_index(); + auto& level_settings = *global_vector_kmeans_tree_index->mutable_level_table_settings(); + auto& posting_settings = *global_vector_kmeans_tree_index->mutable_posting_table_settings(); + auto& vector_settings = *global_vector_kmeans_tree_index->mutable_vector_settings(); + if (GlobalIndexSettings_.size() == 2) { + GlobalIndexSettings_[0].SerializeTo(level_settings); + GlobalIndexSettings_[1].SerializeTo(posting_settings); + } + if (VectorIndexSettings_) { + VectorIndexSettings_->SerializeTo(vector_settings); + } break; + } case EIndexType::Unknown: break; } @@ -2315,6 +2562,9 @@ void TIndexDescription::Out(IOutputStream& o) const { o << ", data_columns: [" << JoinSeq(", ", DataColumns_) << "]"; } + if (VectorIndexSettings_) { + o << ", vector_settings: " << *VectorIndexSettings_ << ""; + } o << " }"; } @@ -2345,6 +2595,38 @@ TChangefeedDescription::TChangefeedDescription(const Ydb::Table::ChangefeedDescr : TChangefeedDescription(FromProto(proto)) {} +TChangefeedDescription::TInitialScanProgress::TInitialScanProgress() + : PartsTotal(0) + , PartsCompleted(0) +{} + +TChangefeedDescription::TInitialScanProgress::TInitialScanProgress(uint32_t total, uint32_t completed) + : PartsTotal(total) + , PartsCompleted(completed) +{} + +TChangefeedDescription::TInitialScanProgress& TChangefeedDescription::TInitialScanProgress::operator+=(const TInitialScanProgress& other) { + PartsTotal += other.PartsTotal; + PartsCompleted += other.PartsCompleted; + return *this; +} + +uint32_t TChangefeedDescription::TInitialScanProgress::GetPartsTotal() const { + return PartsTotal; +} + +uint32_t TChangefeedDescription::TInitialScanProgress::GetPartsCompleted() const { + return PartsCompleted; +} + +float TChangefeedDescription::TInitialScanProgress::GetProgress() const { + if (PartsTotal == 0) { + return 0; + } + + return 100 * float(PartsCompleted) / float(PartsTotal); +} + TChangefeedDescription& TChangefeedDescription::WithVirtualTimestamps() { VirtualTimestamps_ = true; return *this; @@ -2421,6 +2703,10 @@ const std::string& TChangefeedDescription::GetAwsRegion() const { return AwsRegion_; } +const std::optional& TChangefeedDescription::GetInitialScanProgress() const { + return InitialScanProgress_; +} + template TChangefeedDescription TChangefeedDescription::FromProto(const TProto& proto) { EChangefeedMode mode; @@ -2488,6 +2774,13 @@ TChangefeedDescription TChangefeedDescription::FromProto(const TProto& proto) { ret.State_ = EChangefeedState::Unknown; break; } + + if (proto.has_initial_scan_progress()) { + ret.InitialScanProgress_ = std::make_optional( + proto.initial_scan_progress().parts_total(), + proto.initial_scan_progress().parts_completed() + ); + } } for (const auto& [key, value] : proto.attributes()) { @@ -2575,6 +2868,10 @@ void TChangefeedDescription::Out(IOutputStream& o) const { o << ", aws_region: " << AwsRegion_; } + if (InitialScanProgress_) { + o << ", initial_scan_progress: " << InitialScanProgress_->GetProgress() << "%"; + } + o << " }"; } diff --git a/src/client/topic/impl/read_session_impl.h b/src/client/topic/impl/read_session_impl.h index b7464cae4c..e4da1a0530 100644 --- a/src/client/topic/impl/read_session_impl.h +++ b/src/client/topic/impl/read_session_impl.h @@ -140,7 +140,7 @@ class TDeferredActions { } void DeferReadFromProcessor(const typename IProcessor::TPtr& processor, TServerMessage* dst, typename IProcessor::TReadCallback callback); - void DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction task); + void DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction&& task); void DeferAbortSession(TCallbackContextPtr cbContext, TASessionClosedEvent&& closeEvent); void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, NYql::TIssues&& issues); void DeferAbortSession(TCallbackContextPtr cbContext, EStatus statusCode, const std::string& message); @@ -206,6 +206,8 @@ class TDataDecompressionInfo : public std::enable_shared_from_this> partitionStream); + void OnDestroyReadSession(); + bool IsReady() const { return SourceDataNotProcessed == 0; } @@ -303,6 +305,8 @@ class TDataDecompressionInfo : public std::enable_shared_from_this> PartitionStream; @@ -1096,6 +1100,11 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext& deferred); void OnDataDecompressed(i64 sourceSize, i64 estimatedDecompressedSize, i64 decompressedSize, size_t messagesCount, i64 serverBytesSize = 0); @@ -1285,6 +1294,8 @@ class TSingleClusterReadSessionImpl : public TEnableSelfContext BatchInfo; TIntrusivePtr> PartitionStream; }; diff --git a/src/client/topic/impl/read_session_impl.ipp b/src/client/topic/impl/read_session_impl.ipp index fb070e3b7a..5e7d976f68 100644 --- a/src/client/topic/impl/read_session_impl.ipp +++ b/src/client/topic/impl/read_session_impl.ipp @@ -218,6 +218,15 @@ void TRawPartitionStreamEventQueue::DeleteNotReadyTail(TDe swap(ready, NotReady); } +//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// TDecompressionQueueItem + +template +void TSingleClusterReadSessionImpl::TDecompressionQueueItem::OnDestroyReadSession() +{ + BatchInfo->OnDestroyReadSession(); +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TSingleClusterReadSessionImpl @@ -226,6 +235,10 @@ TSingleClusterReadSessionImpl::~TSingleClusterReadSessionI for (auto&& [_, partitionStream] : PartitionStreams) { partitionStream->ClearQueue(); } + + for (auto& e : DecompressionQueue) { + e.OnDestroyReadSession(); + } } template @@ -258,7 +271,7 @@ bool TSingleClusterReadSessionImpl::Reconnect(const TPlain if (!status.Ok()) { LOG_LAZY(Log, TLOG_ERR, GetLogPrefix() << "Got error. Status: " << status.Status - << ". Description: " << IssuesSingleLineString(status.Issues)); + << ". Description: " << IssuesSingleLineString(status.Issues)); } NYdbGrpc::IQueueClientContextPtr delayContext = nullptr; @@ -1565,6 +1578,7 @@ void TSingleClusterReadSessionImpl::OnDecompressionInfoDes template void TSingleClusterReadSessionImpl::OnDataDecompressed(i64 sourceSize, i64 estimatedDecompressedSize, i64 decompressedSize, size_t messagesCount, i64 serverBytesSize) { + TDeferredActions deferred; Y_ABORT_UNLESS(DecompressionTasksInflight > 0); @@ -2512,6 +2526,14 @@ void TDataDecompressionInfo::PlanDecompressionTasks(double } } +template +void TDataDecompressionInfo::OnDestroyReadSession() +{ + for (auto& task : Tasks) { + task.ClearParent(); + } +} + template void TDataDecompressionEvent::TakeData(TIntrusivePtr> partitionStream, std::vector::TMessage>& messages, @@ -2661,19 +2683,23 @@ TDataDecompressionInfo::TDecompressionTask::TDecompression template void TDataDecompressionInfo::TDecompressionTask::operator()() { + auto parent = Parent; + if (!parent) { + return; + } i64 minOffset = Max(); i64 maxOffset = 0; - const i64 partition_id = [this](){ + const i64 partition_id = [parent](){ if constexpr (UseMigrationProtocol) { - return Parent->ServerMessage.partition(); + return parent->ServerMessage.partition(); } else { - return Parent->ServerMessage.partition_session_id(); + return parent->ServerMessage.partition_session_id(); } }(); i64 dataProcessed = 0; size_t messagesProcessed = 0; for (const TMessageRange& messages : Messages) { - auto& batch = *Parent->ServerMessage.mutable_batches(messages.Batch); + auto& batch = *parent->ServerMessage.mutable_batches(messages.Batch); for (size_t i = messages.MessageRange.first; i < messages.MessageRange.second; ++i) { auto& data = *batch.mutable_message_data(i); @@ -2684,7 +2710,7 @@ void TDataDecompressionInfo::TDecompressionTask::operator( try { if constexpr (UseMigrationProtocol) { - if (Parent->DoDecompress + if (parent->DoDecompress && data.codec() != Ydb::PersQueue::V1::CODEC_RAW && data.codec() != Ydb::PersQueue::V1::CODEC_UNSPECIFIED ) { @@ -2694,7 +2720,7 @@ void TDataDecompressionInfo::TDecompressionTask::operator( data.set_codec(Ydb::PersQueue::V1::CODEC_RAW); } } else { - if (Parent->DoDecompress + if (parent->DoDecompress && static_cast(batch.codec()) != Ydb::Topic::CODEC_RAW && static_cast(batch.codec()) != Ydb::Topic::CODEC_UNSPECIFIED ) { @@ -2706,32 +2732,38 @@ void TDataDecompressionInfo::TDecompressionTask::operator( DecompressedSize += data.data().size(); } catch (...) { - Parent->PutDecompressionError(std::current_exception(), messages.Batch, i); + parent->PutDecompressionError(std::current_exception(), messages.Batch, i); data.clear_data(); // Free memory, because we don't count it. - if (auto session = Parent->CbContext->LockShared()) { + if (auto session = parent->CbContext->LockShared()) { session->GetLog() << TLOG_INFO << "Error decompressing data: " << CurrentExceptionMessage(); } } } } - if (auto session = Parent->CbContext->LockShared()) { + if (auto session = parent->CbContext->LockShared()) { LOG_LAZY(session->GetLog(), TLOG_DEBUG, TStringBuilder() << "Decompression task done. Partition/PartitionSessionId: " << partition_id << " (" << minOffset << "-" << maxOffset << ")"); } Y_ASSERT(dataProcessed == SourceDataSize); - Parent->OnDataDecompressed(SourceDataSize, EstimatedDecompressedSize, DecompressedSize, messagesProcessed); + parent->OnDataDecompressed(SourceDataSize, EstimatedDecompressedSize, DecompressedSize, messagesProcessed); - Parent->SourceDataNotProcessed -= dataProcessed; + parent->SourceDataNotProcessed -= dataProcessed; Ready->Ready = true; - if (auto session = Parent->CbContext->LockShared()) { + if (auto session = parent->CbContext->LockShared()) { session->GetEventsQueue()->SignalReadyEvents(PartitionStream); } } +template +void TDataDecompressionInfo::TDecompressionTask::ClearParent() +{ + Parent = nullptr; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TUserRetrievedEventsInfoAccumulator @@ -2769,7 +2801,7 @@ void TDeferredActions::DeferReadFromProcessor(const typena } template -void TDeferredActions::DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction task) { +void TDeferredActions::DeferStartExecutorTask(const typename IAExecutor::TPtr& executor, typename IAExecutor::TFunction&& task) { ExecutorsTasks.emplace_back(executor, std::move(task)); } diff --git a/src/client/topic/impl/topic.cpp b/src/client/topic/impl/topic.cpp index a5b31dd5ea..677a75b96b 100644 --- a/src/client/topic/impl/topic.cpp +++ b/src/client/topic/impl/topic.cpp @@ -395,6 +395,7 @@ TPartitionInfo::TPartitionInfo(const Ydb::Topic::DescribeTopicResult::PartitionI for (const auto& partId : partitionInfo.parent_partition_ids()) { ParentPartitionIds_.push_back(partId); } + if (partitionInfo.has_partition_stats()) { PartitionStats_ = TPartitionStats{partitionInfo.partition_stats()}; } @@ -402,6 +403,14 @@ TPartitionInfo::TPartitionInfo(const Ydb::Topic::DescribeTopicResult::PartitionI if (partitionInfo.has_partition_location()) { PartitionLocation_ = TPartitionLocation{partitionInfo.partition_location()}; } + + if (partitionInfo.has_key_range() && partitionInfo.key_range().has_from_bound()) { + FromBound_ = std::string{partitionInfo.key_range().from_bound()}; + } + + if (partitionInfo.has_key_range() && partitionInfo.key_range().has_to_bound()) { + ToBound_ = std::string{partitionInfo.key_range().to_bound()}; + } } TPartitionInfo::TPartitionInfo(const Ydb::Topic::DescribeConsumerResult::PartitionInfo& partitionInfo) @@ -437,6 +446,14 @@ const std::optional& TPartitionInfo::GetPartitionLocation() return PartitionLocation_; } +const std::vector TPartitionInfo::GetChildPartitionIds() const { + return ChildPartitionIds_; +} + +const std::vector TPartitionInfo::GetParentPartitionIds() const { + return ParentPartitionIds_; +} + bool TPartitionInfo::GetActive() const { return Active_; } @@ -445,6 +462,14 @@ uint64_t TPartitionInfo::GetPartitionId() const { return PartitionId_; } +const std::optional& TPartitionInfo::GetFromBound() const { + return FromBound_; +} + +const std::optional& TPartitionInfo::GetToBound() const { + return ToBound_; +} + //////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // TTopicClient diff --git a/src/client/topic/impl/write_session_impl.cpp b/src/client/topic/impl/write_session_impl.cpp index b0a37ef6b5..1cc47bf517 100644 --- a/src/client/topic/impl/write_session_impl.cpp +++ b/src/client/topic/impl/write_session_impl.cpp @@ -171,7 +171,6 @@ TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStat LOG_LAZY(DbDriverState->Log, TLOG_DEBUG, LogPrefix() << "Write session is aborting and will not restart"); return result; } - LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Got error. " << status.ToDebugString()); SessionEstablished = false; // Keep DirectWriteToPartitionId value on temporary errors. @@ -192,10 +191,11 @@ TWriteSessionImpl::THandleResult TWriteSessionImpl::RestartImpl(const TPlainStat if (nextDelay) { result.StartDelay = *nextDelay; result.DoRestart = true; - LOG_LAZY(DbDriverState->Log, TLOG_WARNING, LogPrefix() << "Write session will restart in " << result.StartDelay); + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Got error. " << status.ToDebugString()); + LOG_LAZY(DbDriverState->Log, TLOG_INFO, LogPrefix() << "Write session will restart in " << result.StartDelay); ResetForRetryImpl(); - } else { + LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Got error. " << status.ToDebugString()); LOG_LAZY(DbDriverState->Log, TLOG_ERR, LogPrefix() << "Write session will not restart after a fatal error"); result.DoStop = true; CheckHandleResultImpl(result); @@ -301,7 +301,15 @@ void TWriteSessionImpl::OnDescribePartition(const TStatus& status, const Ydb::To if (!status.IsSuccess()) { with_lock(Lock) { - handleResult = OnErrorImpl({status.GetStatus(), MakeIssueWithSubIssues("Failed to get partition location", status.GetIssues())}); + if (status.GetStatus() == EStatus::CLIENT_CALL_UNIMPLEMENTED) { + Settings.DirectWriteToPartition_ = false; + handleResult = OnErrorImpl({ + EStatus::UNAVAILABLE, + MakeIssueWithSubIssues("The server does not support direct write, fallback to in-direct write", status.GetIssues()) + }); + } else { + handleResult = OnErrorImpl({status.GetStatus(), MakeIssueWithSubIssues("Failed to get partition location", status.GetIssues())}); + } } ProcessHandleResult(handleResult); return; @@ -846,6 +854,11 @@ void TWriteSessionImpl::OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t co } } } + + for (auto& event : processResult.Events) { + EventsQueue->PushEvent(std::move(event)); + } + if (doRead) ReadFromProcessor(); @@ -860,9 +873,6 @@ void TWriteSessionImpl::OnReadDone(NYdbGrpc::TGrpcStatus&& grpcStatus, size_t co CloseImpl(std::move(errorStatus)); } } - for (auto& event : processResult.Events) { - EventsQueue->PushEvent(std::move(event)); - } if (needSetValue) { InitSeqNoPromise.SetValue(*processResult.InitSeqNo); processResult.HandleResult.DoSetSeqNo = false; // Redundant. Just in case. @@ -991,17 +1001,23 @@ TWriteSessionImpl::TProcessSrvMessageResult TWriteSessionImpl::ProcessServerMess writeStat->PartitionQuotedTime = durationConv(stat.partition_quota_wait_time()); writeStat->TopicQuotedTime = durationConv(stat.topic_quota_wait_time()); - for (size_t messageIndex = 0, endIndex = batchWriteResponse.acks_size(); messageIndex != endIndex; ++messageIndex) { + for (const auto& ack : batchWriteResponse.acks()) { // TODO: Fill writer statistics - auto ack = batchWriteResponse.acks(messageIndex); uint64_t sequenceNumber = ack.seq_no(); - Y_ABORT_UNLESS(ack.has_written() || ack.has_skipped()); - auto msgWriteStatus = ack.has_written() - ? TWriteSessionEvent::TWriteAck::EES_WRITTEN - : (ack.skipped().reason() == Ydb::Topic::StreamWriteMessage_WriteResponse_WriteAck_Skipped_Reason::StreamWriteMessage_WriteResponse_WriteAck_Skipped_Reason_REASON_ALREADY_WRITTEN - ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN - : TWriteSessionEvent::TWriteAck::EES_DISCARDED); + Y_ABORT_UNLESS(ack.has_written() || ack.has_skipped() || ack.has_written_in_tx()); + + TWriteSessionEvent::TWriteAck::EEventState msgWriteStatus; + if (ack.has_written_in_tx()) { + msgWriteStatus = TWriteSessionEvent::TWriteAck::EES_WRITTEN_IN_TX; + } else if (ack.has_written()) { + msgWriteStatus = TWriteSessionEvent::TWriteAck::EES_WRITTEN; + } else { + msgWriteStatus = + (ack.skipped().reason() == Ydb::Topic::StreamWriteMessage_WriteResponse_WriteAck_Skipped_Reason::StreamWriteMessage_WriteResponse_WriteAck_Skipped_Reason_REASON_ALREADY_WRITTEN) + ? TWriteSessionEvent::TWriteAck::EES_ALREADY_WRITTEN + : TWriteSessionEvent::TWriteAck::EES_DISCARDED; + } uint64_t offset = ack.has_written() ? ack.written().offset() : 0; @@ -1361,8 +1377,11 @@ void TWriteSessionImpl::SendImpl() { auto* writeRequest = clientMessage.mutable_write_request(); ui32 prevCodec = 0; + + ui64 currentSize = 0; + // Send blocks while we can without messages reordering. - while (IsReadyToSendNextImpl() && clientMessage.ByteSizeLong() < GetMaxGrpcMessageSize()) { + while (IsReadyToSendNextImpl() && currentSize < GetMaxGrpcMessageSize()) { const auto& block = PackedMessagesToSend.top(); Y_ABORT_UNLESS(block.Valid); if (writeRequest->messages_size() > 0 && prevCodec != block.CodecID) { @@ -1410,6 +1429,8 @@ void TWriteSessionImpl::SendImpl() { moveBlock.Move(block); SentPackedMessage.emplace(std::move(moveBlock)); PackedMessagesToSend.pop(); + + currentSize += writeRequest->ByteSizeLong(); } UpdateTokenIfNeededImpl(); LOG_LAZY(DbDriverState->Log, diff --git a/src/client/topic/ut/local_partition_ut.cpp b/src/client/topic/ut/local_partition_ut.cpp index 68f391e315..a5b6718016 100644 --- a/src/client/topic/ut/local_partition_ut.cpp +++ b/src/client/topic/ut/local_partition_ut.cpp @@ -677,18 +677,19 @@ namespace NYdb::NTopic::NTests { .MessageGroupId(TEST_MESSAGE_GROUP_ID) .DirectWriteToPartition(true); auto writeSession = client.CreateSimpleBlockingWriteSession(writeSettings); - TTestReadSession ReadSession("Session-0", client, 2); + auto ReadSession = NPQ::NTest::CreateTestReadSession({ .Name="Session-0", .Setup=setup, .Sdk = NPQ::NTest::SdkVersion::Topic, .ExpectedMessagesCount = 2 }); - UNIT_ASSERT(writeSession->Write(Msg("message_1.1", 2))); + + UNIT_ASSERT(writeSession->Write(NPQ::NTest::Msg("message_1.1", 2))); ui64 txId = 1006; - SplitPartition(setup, ++txId, 0, "a"); + NPQ::NTest::SplitPartition(setup, ++txId, 0, "a"); - UNIT_ASSERT(writeSession->Write(Msg("message_1.2", 3))); + UNIT_ASSERT(writeSession->Write(NPQ::NTest::Msg("message_1.2", 3))); - ReadSession.WaitAllMessages(); + ReadSession->WaitAllMessages(); - for (const auto& info : ReadSession.Impl->ReceivedMessages) { + for (const auto& info : ReadSession->GetReceivedMessages()) { if (info.Data == "message_1.1") { UNIT_ASSERT_EQUAL(0, info.PartitionId); UNIT_ASSERT_EQUAL(2, info.SeqNo); @@ -724,7 +725,7 @@ namespace NYdb::NTopic::NTests { auto const events = tracingBackend->GetEvents(); UNIT_ASSERT(expected.Matches(events)); - ReadSession.Close(); + ReadSession->Close(); } } } diff --git a/src/client/topic/ut/topic_to_table_ut.cpp b/src/client/topic/ut/topic_to_table_ut.cpp index 2196fbc25e..3373f965dc 100644 --- a/src/client/topic/ut/topic_to_table_ut.cpp +++ b/src/client/topic/ut/topic_to_table_ut.cpp @@ -1898,6 +1898,29 @@ Y_UNIT_TEST_F(WriteToTopic_Demo_27, TFixture) } } +Y_UNIT_TEST_F(WriteToTopic_Demo_28, TFixture) +{ + // The test verifies that the `WriteInflightSize` is correctly considered for the main partition. + // Writing to the service partition does not change the `WriteInflightSize` of the main one. + CreateTopic("topic_A", TEST_CONSUMER); + + NTable::TSession tableSession = CreateTableSession(); + NTable::TTransaction tx = BeginTx(tableSession); + + TString message(16'000, 'a'); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_1, TString(16'000, 'a'), &tx, 0); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_1); + + CommitTx(tx, EStatus::SUCCESS); + + WriteToTopic("topic_A", TEST_MESSAGE_GROUP_ID_2, TString(20'000, 'b'), nullptr, 0); + WaitForAcks("topic_A", TEST_MESSAGE_GROUP_ID_2); + + auto messages = ReadFromTopic("topic_A", TEST_CONSUMER, TDuration::Seconds(2), nullptr, 0); + UNIT_ASSERT_VALUES_EQUAL(messages.size(), 2); +} + } } diff --git a/src/client/types/operation/operation.cpp b/src/client/types/operation/operation.cpp index 92847a087a..c325f465e7 100644 --- a/src/client/types/operation/operation.cpp +++ b/src/client/types/operation/operation.cpp @@ -21,6 +21,8 @@ class TOperation::TImpl { : Id_(operation.id(), true /* allowEmpty */) , Status_(std::move(status)) , Ready_(operation.ready()) + , CreateTime_(ProtoTimestampToInstant(operation.create_time())) + , EndTime_(ProtoTimestampToInstant(operation.end_time())) , Operation_(std::move(operation)) { } @@ -37,6 +39,18 @@ class TOperation::TImpl { return Status_; } + TInstant CreateTime() const { + return CreateTime_; + } + + TInstant EndTime() const { + return EndTime_; + } + + const std::string& CreatedBy() const { + return CreatedBy_; + } + const Ydb::Operations::Operation& GetProto() const { return Operation_; } @@ -45,6 +59,9 @@ class TOperation::TImpl { const TOperationId Id_; const TStatus Status_; const bool Ready_; + const TInstant CreateTime_; + const TInstant EndTime_; + const std::string CreatedBy_; const Ydb::Operations::Operation Operation_; }; @@ -68,6 +85,18 @@ const TStatus& TOperation::Status() const { return Impl_->Status(); } +TInstant TOperation::CreateTime() const { + return Impl_->CreateTime(); +} + +TInstant TOperation::EndTime() const { + return Impl_->EndTime(); +} + +const std::string& TOperation::CreatedBy() const { + return Impl_->CreatedBy(); +} + std::string TOperation::ToString() const { TString result; TStringOutput out(result); @@ -92,4 +121,10 @@ const Ydb::Operations::Operation& TOperation::GetProto() const { return Impl_->GetProto(); } +TInstant ProtoTimestampToInstant(const google::protobuf::Timestamp& timestamp) { + ui64 us = timestamp.seconds() * 1000000; + us += timestamp.nanos() / 1000; + return TInstant::MicroSeconds(us); +} + } // namespace NYdb