Skip to content

Commit d76499b

Browse files
jepett0jepett0
authored
External data sources: local backup restore tests (#14596)
1 parent 0add0a1 commit d76499b

File tree

1 file changed

+159
-16
lines changed

1 file changed

+159
-16
lines changed

‎ydb/services/ydb/backup_ut/ydb_backup_ut.cpp

Lines changed: 159 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
#include <ydb/public/api/protos/draft/ydb_replication.pb.h>
66
#include <ydb/public/api/protos/draft/ydb_view.pb.h>
77
#include <ydb/public/api/protos/ydb_rate_limiter.pb.h>
8+
#include <ydb/public/api/protos/ydb_table.pb.h>
89
#include <ydb/public/lib/ydb_cli/common/recursive_list.h>
910
#include <ydb/public/lib/ydb_cli/dump/dump.h>
1011
#include <ydb/public/lib/yson_value/ydb_yson_value.h>
@@ -14,6 +15,7 @@
1415
#include <ydb-cpp-sdk/client/export/export.h>
1516
#include <ydb-cpp-sdk/client/import/import.h>
1617
#include <ydb-cpp-sdk/client/operation/operation.h>
18+
#include <ydb-cpp-sdk/client/proto/accessor.h>
1719
#include <ydb-cpp-sdk/client/query/client.h>
1820
#include <ydb-cpp-sdk/client/rate_limiter/rate_limiter.h>
1921
#include <ydb-cpp-sdk/client/table/table.h>
@@ -35,6 +37,34 @@ using namespace NYdb::NScheme;
3537
using namespace NYdb::NTable;
3638
using namespace NYdb::NView;
3739

40+
namespace Ydb::Table {
41+
42+
bool operator==(const DescribeExternalDataSourceResult& lhs, const DescribeExternalDataSourceResult& rhs) {
43+
google::protobuf::util::MessageDifferencer differencer;
44+
differencer.IgnoreField(DescribeExternalDataSourceResult::GetDescriptor()->FindFieldByName("self"));
45+
return differencer.Compare(lhs, rhs);
46+
}
47+
48+
bool operator==(const DescribeExternalTableResult& lhs, const DescribeExternalTableResult& rhs) {
49+
google::protobuf::util::MessageDifferencer differencer;
50+
differencer.IgnoreField(DescribeExternalTableResult::GetDescriptor()->FindFieldByName("self"));
51+
return differencer.Compare(lhs, rhs);
52+
}
53+
54+
}
55+
56+
namespace Ydb::RateLimiter {
57+
58+
bool operator==(const HierarchicalDrrSettings& lhs, const HierarchicalDrrSettings& rhs) {
59+
return google::protobuf::util::MessageDifferencer::Equals(lhs, rhs);
60+
}
61+
62+
bool operator==(const MeteringConfig& lhs, const MeteringConfig& rhs) {
63+
return google::protobuf::util::MessageDifferencer::Equals(lhs, rhs);
64+
}
65+
66+
}
67+
3868
namespace NYdb::NTable {
3969

4070
bool operator==(const TValue& lhs, const TValue& rhs) {
@@ -53,13 +83,6 @@ bool operator==(const TKeyRange& lhs, const TKeyRange& rhs) {
5383

5484
namespace NYdb::NRateLimiter {
5585

56-
bool operator==(
57-
const Ydb::RateLimiter::HierarchicalDrrSettings& lhs,
58-
const Ydb::RateLimiter::HierarchicalDrrSettings& rhs
59-
) {
60-
return google::protobuf::util::MessageDifferencer::Equals(lhs, rhs);
61-
}
62-
6386
bool operator==(
6487
const TDescribeResourceResult::THierarchicalDrrProps& lhs,
6588
const TDescribeResourceResult::THierarchicalDrrProps& rhs
@@ -71,13 +94,6 @@ bool operator==(
7194
return left == right;
7295
}
7396

74-
bool operator==(
75-
const Ydb::RateLimiter::MeteringConfig& lhs,
76-
const Ydb::RateLimiter::MeteringConfig& rhs
77-
) {
78-
return google::protobuf::util::MessageDifferencer::Equals(lhs, rhs);
79-
}
80-
8197
bool operator==(const TMeteringConfig& lhs, const TMeteringConfig& rhs) {
8298
Ydb::RateLimiter::MeteringConfig left;
8399
lhs.SerializeTo(left);
@@ -990,6 +1006,87 @@ void TestReplicationSettingsArePreserved(
9901006
checkDescription();
9911007
}
9921008

1009+
Ydb::Table::DescribeExternalDataSourceResult DescribeExternalDataSource(TSession& session, const TString& path) {
1010+
auto result = session.DescribeExternalDataSource(path).ExtractValueSync();
1011+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
1012+
return TProtoAccessor::GetProto(result.GetExternalDataSourceDescription());;
1013+
}
1014+
1015+
void TestExternalDataSourceSettingsArePreserved(
1016+
const char* path, TSession& tableSession, NQuery::TSession& querySession, TBackupFunction&& backup, TRestoreFunction&& restore
1017+
) {
1018+
ExecuteQuery(querySession, Sprintf(R"(
1019+
CREATE EXTERNAL DATA SOURCE `%s` WITH (
1020+
SOURCE_TYPE = "ObjectStorage",
1021+
LOCATION = "192.168.1.1:8123",
1022+
AUTH_METHOD = "NONE"
1023+
);
1024+
)", path
1025+
), true
1026+
);
1027+
const auto originalDescription = DescribeExternalDataSource(tableSession, path);
1028+
1029+
backup();
1030+
1031+
ExecuteQuery(querySession, Sprintf(R"(
1032+
DROP EXTERNAL DATA SOURCE `%s`;
1033+
)", path
1034+
), true
1035+
);
1036+
1037+
restore();
1038+
UNIT_ASSERT_VALUES_EQUAL(
1039+
DescribeExternalDataSource(tableSession, path),
1040+
originalDescription
1041+
);
1042+
}
1043+
1044+
Ydb::Table::DescribeExternalTableResult DescribeExternalTable(TSession& session, const TString& path) {
1045+
auto result = session.DescribeExternalTable(path).ExtractValueSync();
1046+
UNIT_ASSERT_C(result.IsSuccess(), result.GetIssues().ToString());
1047+
return TProtoAccessor::GetProto(result.GetExternalTableDescription());;
1048+
}
1049+
1050+
void TestExternalTableSettingsArePreserved(
1051+
const char* path, const char* externalDataSource, TSession& tableSession, NQuery::TSession& querySession, TBackupFunction&& backup, TRestoreFunction&& restore
1052+
) {
1053+
ExecuteQuery(querySession, Sprintf(R"(
1054+
CREATE EXTERNAL DATA SOURCE `%s` WITH (
1055+
SOURCE_TYPE = "ObjectStorage",
1056+
LOCATION = "192.168.1.1:8123",
1057+
AUTH_METHOD = "NONE"
1058+
);
1059+
1060+
CREATE EXTERNAL TABLE `%s` (
1061+
key Utf8 NOT NULL,
1062+
value Utf8 NOT NULL
1063+
) WITH (
1064+
DATA_SOURCE = "%s",
1065+
LOCATION = "folder",
1066+
FORMAT = "csv_with_names",
1067+
COMPRESSION = "gzip"
1068+
);
1069+
)", externalDataSource, path, externalDataSource
1070+
), true
1071+
);
1072+
const auto originalDescription = DescribeExternalTable(tableSession, path);
1073+
1074+
backup();
1075+
1076+
ExecuteQuery(querySession, Sprintf(R"(
1077+
DROP EXTERNAL TABLE `%s`;
1078+
DROP EXTERNAL DATA SOURCE `%s`;
1079+
)", path, externalDataSource
1080+
), true
1081+
);
1082+
1083+
restore();
1084+
UNIT_ASSERT_VALUES_EQUAL(
1085+
DescribeExternalTable(tableSession, path),
1086+
originalDescription
1087+
);
1088+
}
1089+
9931090
}
9941091

9951092
Y_UNIT_TEST_SUITE(BackupRestore) {
@@ -1398,6 +1495,52 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
13981495
);
13991496
}
14001497

1498+
void TestExternalDataSourceBackupRestore() {
1499+
TKikimrWithGrpcAndRootSchema server;
1500+
server.GetRuntime()->GetAppData().FeatureFlags.SetEnableExternalDataSources(true);
1501+
auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort())));
1502+
TTableClient tableClient(driver);
1503+
auto tableSession = tableClient.CreateSession().ExtractValueSync().GetSession();
1504+
NQuery::TQueryClient queryClient(driver);
1505+
auto querySession = queryClient.GetSession().ExtractValueSync().GetSession();
1506+
TTempDir tempDir;
1507+
const auto& pathToBackup = tempDir.Path();
1508+
1509+
constexpr const char* path = "/Root/externalDataSource";
1510+
1511+
TestExternalDataSourceSettingsArePreserved(
1512+
path,
1513+
tableSession,
1514+
querySession,
1515+
CreateBackupLambda(driver, pathToBackup),
1516+
CreateRestoreLambda(driver, pathToBackup)
1517+
);
1518+
}
1519+
1520+
void TestExternalTableBackupRestore() {
1521+
TKikimrWithGrpcAndRootSchema server;
1522+
server.GetRuntime()->GetAppData().FeatureFlags.SetEnableExternalDataSources(true);
1523+
auto driver = TDriver(TDriverConfig().SetEndpoint(Sprintf("localhost:%u", server.GetPort())));
1524+
TTableClient tableClient(driver);
1525+
auto tableSession = tableClient.CreateSession().ExtractValueSync().GetSession();
1526+
NQuery::TQueryClient queryClient(driver);
1527+
auto querySession = queryClient.GetSession().ExtractValueSync().GetSession();
1528+
TTempDir tempDir;
1529+
const auto& pathToBackup = tempDir.Path();
1530+
1531+
constexpr const char* path = "/Root/externalTable";
1532+
constexpr const char* externalDataSource = "/Root/externalDataSource";
1533+
1534+
TestExternalTableSettingsArePreserved(
1535+
path,
1536+
externalDataSource,
1537+
tableSession,
1538+
querySession,
1539+
CreateBackupLambda(driver, pathToBackup),
1540+
CreateRestoreLambda(driver, pathToBackup)
1541+
);
1542+
}
1543+
14011544
Y_UNIT_TEST_ALL_PROTO_ENUM_VALUES(TestAllSchemeObjectTypes, NKikimrSchemeOp::EPathType) {
14021545
using namespace NKikimrSchemeOp;
14031546

@@ -1424,9 +1567,9 @@ Y_UNIT_TEST_SUITE(BackupRestore) {
14241567
case EPathTypeTransfer:
14251568
break; // https://github.com/ydb-platform/ydb/issues/10436
14261569
case EPathTypeExternalTable:
1427-
break; // https://github.com/ydb-platform/ydb/issues/10438
1570+
return TestExternalTableBackupRestore();
14281571
case EPathTypeExternalDataSource:
1429-
break; // https://github.com/ydb-platform/ydb/issues/10439
1572+
return TestExternalDataSourceBackupRestore();
14301573
case EPathTypeResourcePool:
14311574
break; // https://github.com/ydb-platform/ydb/issues/10440
14321575
case EPathTypeKesus:

0 commit comments

Comments
 (0)