Skip to content

Commit a10a402

Browse files
jepett0jepett0
authored
External data sources: dump (#14577)
1 parent 0c292ea commit a10a402

File tree

6 files changed

+165
-8
lines changed

6 files changed

+165
-8
lines changed

‎ydb/core/grpc_services/rpc_describe_external_data_source.cpp

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,9 @@ Ydb::Table::DescribeExternalDataSourceResult Convert(const TDirEntry& inSelf, co
8181
out.set_source_type(inDesc.GetSourceType());
8282
out.set_location(inDesc.GetLocation());
8383
auto& properties = *out.mutable_properties();
84-
properties = inDesc.GetProperties().GetProperties();
84+
for (const auto& [key, value] : inDesc.GetProperties().GetProperties()) {
85+
properties[to_upper(key)] = value;
86+
}
8587
Convert(inDesc.GetAuth(), properties);
8688
return out;
8789
}

‎ydb/core/grpc_services/rpc_describe_external_table.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ bool ConvertContent(
4444
for (const auto& item : items) {
4545
json.AppendValue(item);
4646
}
47-
out[key] = WriteJson(json, false);
47+
out[to_upper(key)] = WriteJson(json, false);
4848
}
4949
} catch (...) {
5050
issues.AddIssue(TStringBuilder() << "Cannot unpack the content of an external table of type: " << sourceType

‎ydb/library/backup/backup.cpp

Lines changed: 139 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
#include <yql/essentials/sql/v1/format/sql_format.h>
3434

3535
#include <library/cpp/containers/stack_vector/stack_vec.h>
36+
#include <library/cpp/json/json_reader.h>
3637
#include <library/cpp/regex/pcre/regexp.h>
3738
#include <library/cpp/string_utils/quote/quote.h>
3839

@@ -55,6 +56,7 @@
5556
#include <google/protobuf/text_format.h>
5657

5758
#include <format>
59+
#include <ranges>
5860

5961
namespace NYdb::NBackup {
6062

@@ -788,6 +790,131 @@ void BackupReplication(
788790
BackupPermissions(driver, dbPath, fsBackupFolder);
789791
}
790792

793+
namespace {
794+
795+
Ydb::Table::DescribeExternalDataSourceResult DescribeExternalDataSource(TDriver driver, const TString& path) {
796+
NTable::TTableClient client(driver);
797+
Ydb::Table::DescribeExternalDataSourceResult description;
798+
auto status = client.RetryOperationSync([&](NTable::TSession session) {
799+
auto result = session.DescribeExternalDataSource(path).ExtractValueSync();
800+
if (result.IsSuccess()) {
801+
description = TProtoAccessor::GetProto(result.GetExternalDataSourceDescription());
802+
}
803+
return result;
804+
});
805+
VerifyStatus(status, "describe external data source");
806+
return description;
807+
}
808+
809+
std::string ToString(std::string_view key, std::string_view value) {
810+
// indented to follow the default YQL formatting
811+
return std::format(R"( {} = "{}")", key, value);
812+
}
813+
814+
namespace NExternalDataSource {
815+
816+
std::string PropertyToString(const std::pair<TProtoStringType, TProtoStringType>& property) {
817+
const auto& [key, value] = property;
818+
return ToString(key, value);
819+
}
820+
821+
}
822+
823+
TString BuildCreateExternalDataSourceQuery(const Ydb::Table::DescribeExternalDataSourceResult& description) {
824+
return std::format(
825+
"CREATE EXTERNAL DATA SOURCE IF NOT EXISTS `{}` WITH (\n{},\n{}{}\n);",
826+
description.self().name().c_str(),
827+
ToString("SOURCE_TYPE", description.source_type()),
828+
ToString("LOCATION", description.location()),
829+
description.properties().empty()
830+
? ""
831+
: std::string(",\n") +
832+
JoinSeq(",\n", std::views::transform(description.properties(), NExternalDataSource::PropertyToString)).c_str()
833+
);
834+
}
835+
836+
}
837+
838+
void BackupExternalDataSource(TDriver driver, const TString& dbPath, const TFsPath& fsBackupFolder) {
839+
Y_ENSURE(!dbPath.empty());
840+
LOG_I("Backup external data source " << dbPath.Quote() << " to " << fsBackupFolder.GetPath().Quote());
841+
842+
const auto description = DescribeExternalDataSource(driver, dbPath);
843+
const auto creationQuery = BuildCreateExternalDataSourceQuery(description);
844+
845+
WriteCreationQueryToFile(creationQuery, fsBackupFolder, NDump::NFiles::CreateExternalDataSource());
846+
BackupPermissions(driver, dbPath, fsBackupFolder);
847+
}
848+
849+
namespace {
850+
851+
Ydb::Table::DescribeExternalTableResult DescribeExternalTable(TDriver driver, const TString& path) {
852+
NTable::TTableClient client(driver);
853+
Ydb::Table::DescribeExternalTableResult description;
854+
auto status = client.RetryOperationSync([&](NTable::TSession session) {
855+
auto result = session.DescribeExternalTable(path).ExtractValueSync();
856+
if (result.IsSuccess()) {
857+
description = TProtoAccessor::GetProto(result.GetExternalTableDescription());
858+
}
859+
return result;
860+
});
861+
VerifyStatus(status, "describe external table");
862+
return description;
863+
}
864+
865+
namespace NExternalTable {
866+
867+
std::string PropertyToString(const std::pair<TProtoStringType, TProtoStringType>& property) {
868+
const auto& [key, json] = property;
869+
const auto items = NJson::ReadJsonFastTree(json).GetArray();
870+
Y_ENSURE(!items.empty(), "Empty items for an external table property: " << key);
871+
if (items.size() == 1) {
872+
return ToString(key, items.front().GetString());
873+
} else {
874+
return ToString(key, std::format("[{}]", JoinSeq(", ", items).c_str()));
875+
}
876+
}
877+
878+
}
879+
880+
std::string ColumnToString(const Ydb::Table::ColumnMeta& column) {
881+
const auto& type = column.type();
882+
const bool notNull = !type.has_optional_type() || (type.has_pg_type() && column.not_null());
883+
return std::format(
884+
" {} {}{}",
885+
column.name().c_str(),
886+
TType(type).ToString(),
887+
notNull ? " NOT NULL" : ""
888+
);
889+
}
890+
891+
TString BuildCreateExternalTableQuery(const Ydb::Table::DescribeExternalTableResult& description) {
892+
return std::format(
893+
"CREATE EXTERNAL TABLE IF NOT EXISTS `{}` (\n{}\n) WITH (\n{},\n{}{}\n);",
894+
description.self().name().c_str(),
895+
JoinSeq(",\n", std::views::transform(description.columns(), ColumnToString)).c_str(),
896+
ToString("DATA_SOURCE", description.data_source_path()),
897+
ToString("LOCATION", description.location()),
898+
description.content().empty()
899+
? ""
900+
: std::string(",\n") +
901+
JoinSeq(",\n", std::views::transform(description.content(), NExternalTable::PropertyToString)).c_str()
902+
);
903+
}
904+
905+
}
906+
907+
void BackupExternalTable(TDriver driver, const TString& dbPath, const TFsPath& fsBackupFolder) {
908+
Y_ENSURE(!dbPath.empty());
909+
LOG_I("Backup external table " << dbPath.Quote() << " to " << fsBackupFolder.GetPath().Quote());
910+
911+
const auto description = DescribeExternalTable(driver, dbPath);
912+
const auto creationQuery = BuildCreateExternalTableQuery(description);
913+
914+
WriteCreationQueryToFile(creationQuery, fsBackupFolder, NDump::NFiles::CreateExternalTable());
915+
BackupPermissions(driver, dbPath, fsBackupFolder);
916+
}
917+
791918
void CreateClusterDirectory(const TDriver& driver, const TString& path, bool rootBackupDir = false) {
792919
if (rootBackupDir) {
793920
LOG_I("Create temporary directory " << path.Quote() << " in database");
@@ -837,11 +964,11 @@ void BackupFolderImpl(TDriver driver, const TString& database, const TString& db
837964
bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy, bool preservePoolKinds, bool ordered,
838965
NYql::TIssues& issues
839966
) {
840-
TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways);
967+
TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways).Close();
841968

842969
TMap<TString, TAsyncStatus> copiedTablesStatuses;
843970
TVector<NTable::TCopyItem> tablesToCopy;
844-
// Copy all tables to temporal folder
971+
// Copy all tables to temporal folder and backup other scheme objects along the way.
845972
{
846973
TDbIterator<ETraverseType::Preordering> dbIt(driver, dbPrefix);
847974
while (dbIt) {
@@ -884,6 +1011,12 @@ void BackupFolderImpl(TDriver driver, const TString& database, const TString& db
8841011
if (dbIt.IsReplication()) {
8851012
BackupReplication(driver, database, dbIt.GetTraverseRoot(), dbIt.GetRelPath(), childFolderPath);
8861013
}
1014+
if (dbIt.IsExternalDataSource()) {
1015+
BackupExternalDataSource(driver, dbIt.GetFullPath(), childFolderPath);
1016+
}
1017+
if (dbIt.IsExternalTable()) {
1018+
BackupExternalTable(driver, dbIt.GetFullPath(), childFolderPath);
1019+
}
8871020
dbIt.Next();
8881021
}
8891022
}
@@ -1040,7 +1173,7 @@ TAdmins FindAdmins(TDriver driver, const TString& dbPath) {
10401173
struct TBackupDatabaseSettings {
10411174
bool WithRegularUsers = false;
10421175
bool WithContent = false;
1043-
TString TemporalBackupPostfix;
1176+
TString TemporalBackupPostfix = "";
10441177
};
10451178

10461179
void BackupUsers(TDriver driver, const TString& dbPath, const TFsPath& folderPath, const THashSet<TString>& filter = {}) {
@@ -1165,7 +1298,7 @@ void BackupDatabaseImpl(TDriver driver, const TString& dbPath, const TFsPath& fo
11651298
Ydb::Cms::CreateDatabaseRequest proto;
11661299
status.SerializeTo(proto);
11671300
WriteProtoToFile(proto, folderPath, NDump::NFiles::Database());
1168-
1301+
11691302
if (!settings.WithRegularUsers) {
11701303
TAdmins admins = FindAdmins(driver, dbPath);
11711304
BackupUsers(driver, dbPath, folderPath, admins.UserSids);
@@ -1308,7 +1441,7 @@ void BackupDatabase(const TDriver& driver, const TString& database, TFsPath fold
13081441

13091442
try {
13101443
NYql::TIssues issues;
1311-
TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways);
1444+
TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways).Close();
13121445

13131446
BackupDatabaseImpl(driver, database, folderPath, {
13141447
.WithRegularUsers = true,
@@ -1339,7 +1472,7 @@ void BackupCluster(const TDriver& driver, TFsPath folderPath) {
13391472

13401473
try {
13411474
NYql::TIssues issues;
1342-
TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways);
1475+
TFile(folderPath.Child(NDump::NFiles::Incomplete().FileName), CreateAlways).Close();
13431476

13441477
BackupClusterRoot(driver, folderPath);
13451478
auto databases = ListDatabases(driver);

‎ydb/library/backup/db_iterator.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -161,6 +161,14 @@ class TDbIterator {
161161
return GetCurrentNode()->Type == NScheme::ESchemeEntryType::Replication;
162162
}
163163

164+
bool IsExternalDataSource() const {
165+
return GetCurrentNode()->Type == NScheme::ESchemeEntryType::ExternalDataSource;
166+
}
167+
168+
bool IsExternalTable() const {
169+
return GetCurrentNode()->Type == NScheme::ESchemeEntryType::ExternalTable;
170+
}
171+
164172
bool IsListed() const {
165173
return NextNodes.front().IsListed;
166174
}

‎ydb/public/lib/ydb_cli/dump/files/files.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@ enum EFilesType {
1919
CREATE_GROUP,
2020
ALTER_GROUP,
2121
CREATE_ASYNC_REPLICATION,
22+
CREATE_EXTERNAL_DATA_SOURCE,
23+
CREATE_EXTERNAL_TABLE,
2224
};
2325

2426
static constexpr TFileInfo FILES_INFO[] = {
@@ -38,6 +40,8 @@ static constexpr TFileInfo FILES_INFO[] = {
3840
{"create_group.sql", "groups"},
3941
{"alter_group.sql", "group members"},
4042
{"create_async_replication.sql", "async replication"},
43+
{"create_external_data_source.sql", "external data source"},
44+
{"create_external_table.sql", "external table"},
4145
};
4246

4347
const TFileInfo& TableScheme() {
@@ -104,4 +108,12 @@ const TFileInfo& CreateAsyncReplication() {
104108
return FILES_INFO[CREATE_ASYNC_REPLICATION];
105109
}
106110

111+
const TFileInfo& CreateExternalDataSource() {
112+
return FILES_INFO[CREATE_EXTERNAL_DATA_SOURCE];
113+
}
114+
115+
const TFileInfo& CreateExternalTable() {
116+
return FILES_INFO[CREATE_EXTERNAL_TABLE];
117+
}
118+
107119
} // NYdb::NDump::NFiles

‎ydb/public/lib/ydb_cli/dump/files/files.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,5 +23,7 @@ const TFileInfo& CreateUser();
2323
const TFileInfo& CreateGroup();
2424
const TFileInfo& AlterGroup();
2525
const TFileInfo& CreateAsyncReplication();
26+
const TFileInfo& CreateExternalDataSource();
27+
const TFileInfo& CreateExternalTable();
2628

2729
} // NYdb::NDump:NFiles

0 commit comments

Comments
 (0)