Skip to content

Commit 102877c

Browse files
jepett0jepett0
committed
Moved commit "Extend index tables' partitioning settings to be able to restore them from a backup with the same partitioning as before" from ydb repo
1 parent edcad84 commit 102877c

File tree

2 files changed

+131
-50
lines changed

2 files changed

+131
-50
lines changed

‎include/ydb-cpp-sdk/client/table/table.h

Lines changed: 56 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@ class CreateTableRequest;
2121
class Changefeed;
2222
class ChangefeedDescription;
2323
class DescribeTableResult;
24+
class ExplicitPartitions;
25+
class GlobalIndexSettings;
2426
class PartitioningSettings;
2527
class DateTypeColumnModeSettings;
2628
class TtlSettings;
@@ -146,17 +148,67 @@ struct TAlterTableColumn {
146148

147149
////////////////////////////////////////////////////////////////////////////////
148150

151+
//! Represents table partitioning settings
152+
class TPartitioningSettings {
153+
public:
154+
TPartitioningSettings();
155+
explicit TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto);
156+
157+
const Ydb::Table::PartitioningSettings& GetProto() const;
158+
159+
std::optional<bool> GetPartitioningBySize() const;
160+
std::optional<bool> GetPartitioningByLoad() const;
161+
uint64_t GetPartitionSizeMb() const;
162+
uint64_t GetMinPartitionsCount() const;
163+
uint64_t GetMaxPartitionsCount() const;
164+
165+
private:
166+
class TImpl;
167+
std::shared_ptr<TImpl> Impl_;
168+
};
169+
170+
struct TExplicitPartitions {
171+
using TSelf = TExplicitPartitions;
172+
173+
FLUENT_SETTING_VECTOR(TValue, SplitPoints);
174+
175+
template <typename TProto>
176+
static TExplicitPartitions FromProto(const TProto& proto);
177+
178+
void SerializeTo(Ydb::Table::ExplicitPartitions& proto) const;
179+
};
180+
181+
struct TGlobalIndexSettings {
182+
using TUniformOrExplicitPartitions = std::variant<std::monostate, uint64_t, TExplicitPartitions>;
183+
184+
TPartitioningSettings PartitioningSettings;
185+
TUniformOrExplicitPartitions Partitions;
186+
187+
template <typename TProto>
188+
static TGlobalIndexSettings FromProto(const TProto& proto);
189+
190+
void SerializeTo(Ydb::Table::GlobalIndexSettings& proto) const;
191+
};
192+
149193
//! Represents index description
150194
class TIndexDescription {
151195
friend class NYdb::TProtoAccessor;
152196

153197
public:
154198
TIndexDescription(
155-
const std::string& name, EIndexType type,
199+
const std::string& name,
200+
EIndexType type,
156201
const std::vector<std::string>& indexColumns,
157-
const std::vector<std::string>& dataColumns = std::vector<std::string>());
202+
const std::vector<std::string>& dataColumns = {},
203+
const TGlobalIndexSettings& settings = {}
204+
);
158205

159-
TIndexDescription(const std::string& name, const std::vector<std::string>& indexColumns, const std::vector<std::string>& dataColumns = std::vector<std::string>());
206+
TIndexDescription(
207+
const std::string& name,
208+
const std::vector<std::string>& indexColumns,
209+
const std::vector<std::string>& dataColumns = {},
210+
const TGlobalIndexSettings& settings = {}
211+
);
160212

161213
const std::string& GetIndexName() const;
162214
EIndexType GetIndexType() const;
@@ -180,6 +232,7 @@ class TIndexDescription {
180232
EIndexType IndexType_;
181233
std::vector<std::string> IndexColumns_;
182234
std::vector<std::string> DataColumns_;
235+
TGlobalIndexSettings GlobalIndexSettings_;
183236
uint64_t SizeBytes = 0;
184237
};
185238

@@ -423,25 +476,6 @@ class TColumnFamilyDescription {
423476
std::shared_ptr<TImpl> Impl_;
424477
};
425478

426-
//! Represents table partitioning settings
427-
class TPartitioningSettings {
428-
public:
429-
TPartitioningSettings();
430-
explicit TPartitioningSettings(const Ydb::Table::PartitioningSettings& proto);
431-
432-
const Ydb::Table::PartitioningSettings& GetProto() const;
433-
434-
std::optional<bool> GetPartitioningBySize() const;
435-
std::optional<bool> GetPartitioningByLoad() const;
436-
uint64_t GetPartitionSizeMb() const;
437-
uint64_t GetMinPartitionsCount() const;
438-
uint64_t GetMaxPartitionsCount() const;
439-
440-
private:
441-
class TImpl;
442-
std::shared_ptr<TImpl> Impl_;
443-
};
444-
445479
//! Represents table read replicas settings
446480
class TReadReplicasSettings {
447481
public:
@@ -1219,12 +1253,6 @@ struct TStoragePolicy {
12191253
FLUENT_SETTING_VECTOR(TColumnFamilyPolicy, ColumnFamilies);
12201254
};
12211255

1222-
struct TExplicitPartitions {
1223-
using TSelf = TExplicitPartitions;
1224-
1225-
FLUENT_SETTING_VECTOR(TValue, SplitPoints);
1226-
};
1227-
12281256
struct TPartitioningPolicy {
12291257
using TSelf = TPartitioningPolicy;
12301258

‎src/client/table/table.cpp

Lines changed: 75 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,24 @@ static void SerializeTo(const TRenameIndex& rename, Ydb::Table::RenameIndexItem&
251251
proto.set_replace_destination(rename.ReplaceDestination_);
252252
}
253253

254+
template <typename TProto>
255+
TExplicitPartitions TExplicitPartitions::FromProto(const TProto& proto) {
256+
TExplicitPartitions out;
257+
for (const auto& splitPoint : proto.split_points()) {
258+
TValue value(TType(splitPoint.type()), splitPoint.value());
259+
out.AppendSplitPoints(value);
260+
}
261+
return out;
262+
}
263+
264+
void TExplicitPartitions::SerializeTo(Ydb::Table::ExplicitPartitions& proto) const {
265+
for (const auto& splitPoint : SplitPoints_) {
266+
auto* boundary = proto.add_split_points();
267+
boundary->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType()));
268+
boundary->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint));
269+
}
270+
}
271+
254272
class TTableDescription::TImpl {
255273
using EUnit = TValueSinceUnixEpochModeSettings::EUnit;
256274

@@ -419,13 +437,7 @@ class TTableDescription::TImpl {
419437
break;
420438

421439
case Ydb::Table::CreateTableRequest::kPartitionAtKeys: {
422-
TExplicitPartitions partitionAtKeys;
423-
for (const auto& splitPoint : request.partition_at_keys().split_points()) {
424-
TValue value(TType(splitPoint.type()), splitPoint.value());
425-
partitionAtKeys.AppendSplitPoints(value);
426-
}
427-
428-
SetPartitionAtKeys(partitionAtKeys);
440+
SetPartitionAtKeys(TExplicitPartitions::FromProto(request.partition_at_keys()));
429441
break;
430442
}
431443

@@ -922,12 +934,7 @@ void TTableDescription::SerializeTo(Ydb::Table::CreateTableRequest& request) con
922934
}
923935

924936
if (const auto& partitionAtKeys = Impl_->GetPartitionAtKeys()) {
925-
auto* borders = request.mutable_partition_at_keys();
926-
for (const auto& splitPoint : partitionAtKeys->SplitPoints_) {
927-
auto* border = borders->add_split_points();
928-
border->mutable_type()->CopyFrom(TProtoAccessor::GetProto(splitPoint.GetType()));
929-
border->mutable_value()->CopyFrom(TProtoAccessor::GetProto(splitPoint));
930-
}
937+
partitionAtKeys->SerializeTo(*request.mutable_partition_at_keys());
931938
} else if (Impl_->GetProto().shard_key_bounds_size()) {
932939
request.mutable_partition_at_keys()->mutable_split_points()->CopyFrom(Impl_->GetProto().shard_key_bounds());
933940
}
@@ -2204,16 +2211,25 @@ bool TRenameItem::ReplaceDestination() const {
22042211

22052212
////////////////////////////////////////////////////////////////////////////////
22062213

2207-
TIndexDescription::TIndexDescription(const std::string& name, EIndexType type,
2208-
const std::vector<std::string>& indexColumns, const std::vector<std::string>& dataColumns)
2209-
: IndexName_(name)
2214+
TIndexDescription::TIndexDescription(
2215+
const std::string& name,
2216+
EIndexType type,
2217+
const std::vector<std::string>& indexColumns,
2218+
const std::vector<std::string>& dataColumns,
2219+
const TGlobalIndexSettings& settings
2220+
) : IndexName_(name)
22102221
, IndexType_(type)
22112222
, IndexColumns_(indexColumns)
22122223
, DataColumns_(dataColumns)
2224+
, GlobalIndexSettings_(settings)
22132225
{}
22142226

2215-
TIndexDescription::TIndexDescription(const std::string& name, const std::vector<std::string>& indexColumns, const std::vector<std::string>& dataColumns)
2216-
: TIndexDescription(name, EIndexType::GlobalSync, indexColumns, dataColumns)
2227+
TIndexDescription::TIndexDescription(
2228+
const std::string& name,
2229+
const std::vector<std::string>& indexColumns,
2230+
const std::vector<std::string>& dataColumns,
2231+
const TGlobalIndexSettings& settings
2232+
) : TIndexDescription(name, EIndexType::GlobalSync, indexColumns, dataColumns, settings)
22172233
{}
22182234

22192235
TIndexDescription::TIndexDescription(const Ydb::Table::TableIndex& tableIndex)
@@ -2244,31 +2260,68 @@ uint64_t TIndexDescription::GetSizeBytes() const {
22442260
return SizeBytes;
22452261
}
22462262

2263+
template <typename TProto>
2264+
TGlobalIndexSettings TGlobalIndexSettings::FromProto(const TProto& proto) {
2265+
auto partitionsFromProto = [](const auto& proto) -> TUniformOrExplicitPartitions {
2266+
switch (proto.partitions_case()) {
2267+
case TProto::kUniformPartitions:
2268+
return proto.uniform_partitions();
2269+
case TProto::kPartitionAtKeys:
2270+
return TExplicitPartitions::FromProto(proto.partition_at_keys());
2271+
default:
2272+
return {};
2273+
}
2274+
};
2275+
2276+
return {
2277+
.PartitioningSettings = TPartitioningSettings(proto.partitioning_settings()),
2278+
.Partitions = partitionsFromProto(proto)
2279+
};
2280+
}
2281+
2282+
void TGlobalIndexSettings::SerializeTo(Ydb::Table::GlobalIndexSettings& settings) const {
2283+
*settings.mutable_partitioning_settings() = PartitioningSettings.GetProto();
2284+
2285+
auto variantVisitor = [&settings](auto&& partitions) {
2286+
using T = std::decay_t<decltype(partitions)>;
2287+
if constexpr (std::is_same_v<T, uint64_t>) {
2288+
settings.set_uniform_partitions(partitions);
2289+
} else if constexpr (std::is_same_v<T, TExplicitPartitions>) {
2290+
partitions.SerializeTo(*settings.mutable_partition_at_keys());
2291+
}
2292+
};
2293+
std::visit(std::move(variantVisitor), Partitions);
2294+
}
2295+
22472296
template <typename TProto>
22482297
TIndexDescription TIndexDescription::FromProto(const TProto& proto) {
22492298
EIndexType type;
22502299
std::vector<std::string> indexColumns;
22512300
std::vector<std::string> dataColumns;
2301+
TGlobalIndexSettings globalIndexSettings;
22522302

22532303
indexColumns.assign(proto.index_columns().begin(), proto.index_columns().end());
22542304
dataColumns.assign(proto.data_columns().begin(), proto.data_columns().end());
22552305

22562306
switch (proto.type_case()) {
22572307
case TProto::kGlobalIndex:
22582308
type = EIndexType::GlobalSync;
2309+
globalIndexSettings = TGlobalIndexSettings::FromProto(proto.global_index().settings());
22592310
break;
22602311
case TProto::kGlobalAsyncIndex:
22612312
type = EIndexType::GlobalAsync;
2313+
globalIndexSettings = TGlobalIndexSettings::FromProto(proto.global_async_index().settings());
22622314
break;
22632315
case TProto::kGlobalUniqueIndex:
22642316
type = EIndexType::GlobalUnique;
2317+
globalIndexSettings = TGlobalIndexSettings::FromProto(proto.global_unique_index().settings());
22652318
break;
22662319
default: // fallback to global sync
22672320
type = EIndexType::GlobalSync;
22682321
break;
22692322
}
22702323

2271-
auto result = TIndexDescription(proto.name(), type, indexColumns, dataColumns);
2324+
auto result = TIndexDescription(proto.name(), type, indexColumns, dataColumns, globalIndexSettings);
22722325
if constexpr (std::is_same_v<TProto, Ydb::Table::TableIndexDescription>) {
22732326
result.SizeBytes = proto.size_bytes();
22742327
}
@@ -2286,13 +2339,13 @@ void TIndexDescription::SerializeTo(Ydb::Table::TableIndex& proto) const {
22862339

22872340
switch (IndexType_) {
22882341
case EIndexType::GlobalSync:
2289-
*proto.mutable_global_index() = Ydb::Table::GlobalIndex();
2342+
GlobalIndexSettings_.SerializeTo(*proto.mutable_global_index()->mutable_settings());
22902343
break;
22912344
case EIndexType::GlobalAsync:
2292-
*proto.mutable_global_async_index() = Ydb::Table::GlobalAsyncIndex();
2345+
GlobalIndexSettings_.SerializeTo(*proto.mutable_global_async_index()->mutable_settings());
22932346
break;
22942347
case EIndexType::GlobalUnique:
2295-
*proto.mutable_global_unique_index() = Ydb::Table::GlobalUniqueIndex();
2348+
GlobalIndexSettings_.SerializeTo(*proto.mutable_global_unique_index()->mutable_settings());
22962349
break;
22972350
case EIndexType::Unknown:
22982351
break;

0 commit comments

Comments
 (0)