Skip to content

Commit 5876eb7

Browse files
CyberROFLCyberROFL
authored
backup & restore async repl: rewrite paths (#14500)
1 parent 76fa0bc commit 5876eb7

File tree

5 files changed

+88
-41
lines changed

5 files changed

+88
-41
lines changed

‎ydb/library/backup/backup.cpp

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -725,7 +725,12 @@ inline TString Interval(const TDuration& value) {
725725
return TStringBuilder() << "Interval('PT" << value.Seconds() << "S')";
726726
}
727727

728-
TString BuildCreateReplicationQuery(const TString& name, const NReplication::TReplicationDescription& desc) {
728+
TString BuildCreateReplicationQuery(
729+
const TString& db,
730+
const TString& backupRoot,
731+
const TString& name,
732+
const NReplication::TReplicationDescription& desc)
733+
{
729734
TVector<TString> targets(::Reserve(desc.GetItems().size()));
730735
for (const auto& item : desc.GetItems()) {
731736
if (!item.DstPath.ends_with("/indexImplTable")) { // TODO(ilnaz): get rid of this hack
@@ -752,18 +757,29 @@ TString BuildCreateReplicationQuery(const TString& name, const NReplication::TRe
752757
opts.push_back(BuildOption("COMMIT_INTERVAL", Interval(desc.GetGlobalConsistency().GetCommitInterval())));
753758
}
754759

755-
return std::format("CREATE ASYNC REPLICATION `{}`\nFOR\n{}\nWITH (\n{}\n);",
756-
name.c_str(), JoinSeq(",\n", targets).c_str(), JoinSeq(",\n", opts).c_str());
760+
return std::format(
761+
"-- database: \"{}\"\n"
762+
"-- backup root: \"{}\"\n"
763+
"CREATE ASYNC REPLICATION `{}`\nFOR\n{}\nWITH (\n{}\n);",
764+
db.c_str(), backupRoot.c_str(), name.c_str(), JoinSeq(",\n", targets).c_str(), JoinSeq(",\n", opts).c_str());
757765
}
758766

759767
}
760768

761-
void BackupReplication(TDriver driver, const TString& dbPath, const TFsPath& fsBackupFolder) {
762-
Y_ENSURE(!dbPath.empty());
769+
void BackupReplication(
770+
TDriver driver,
771+
const TString& db,
772+
const TString& dbBackupRoot,
773+
const TString& dbPathRelativeToBackupRoot,
774+
const TFsPath& fsBackupFolder)
775+
{
776+
Y_ENSURE(!dbPathRelativeToBackupRoot.empty());
777+
const auto dbPath = JoinDatabasePath(dbBackupRoot, dbPathRelativeToBackupRoot);
778+
763779
LOG_I("Backup async replication " << dbPath.Quote() << " to " << fsBackupFolder.GetPath().Quote());
764780

765781
const auto desc = DescribeReplication(driver, dbPath);
766-
const auto creationQuery = BuildCreateReplicationQuery(fsBackupFolder.GetName(), desc);
782+
const auto creationQuery = BuildCreateReplicationQuery(db, dbBackupRoot, fsBackupFolder.GetName(), desc);
767783

768784
WriteCreationQueryToFile(creationQuery, fsBackupFolder, NDump::NFiles::CreateAsyncReplication());
769785
BackupPermissions(driver, dbPath, fsBackupFolder);
@@ -813,7 +829,7 @@ static void MaybeCreateEmptyFile(const TFsPath& folderPath) {
813829
}
814830
}
815831

816-
void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& backupPrefix,
832+
void BackupFolderImpl(TDriver driver, const TString& database, const TString& dbPrefix, const TString& backupPrefix,
817833
const TFsPath folderPath, const TVector<TRegExMatch>& exclusionPatterns,
818834
bool schemaOnly, bool useConsistentCopyTable, bool avoidCopy, bool preservePoolKinds, bool ordered,
819835
NYql::TIssues& issues
@@ -863,7 +879,7 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba
863879
BackupCoordinationNode(driver, dbIt.GetFullPath(), childFolderPath);
864880
}
865881
if (dbIt.IsReplication()) {
866-
BackupReplication(driver, dbIt.GetFullPath(), childFolderPath);
882+
BackupReplication(driver, database, dbIt.GetTraverseRoot(), dbIt.GetRelPath(), childFolderPath);
867883
}
868884
dbIt.Next();
869885
}
@@ -1170,6 +1186,7 @@ void BackupDatabaseImpl(TDriver driver, const TString& dbPath, const TFsPath& fo
11701186
BackupFolderImpl(
11711187
driver,
11721188
dbPath,
1189+
dbPath,
11731190
tmpDbFolder,
11741191
folderPath,
11751192
/* exclusionPatterns */ {},
@@ -1251,7 +1268,7 @@ void BackupFolder(const TDriver& driver, const TString& database, const TString&
12511268
}
12521269

12531270
NYql::TIssues issues;
1254-
BackupFolderImpl(driver, dbPrefix, tmpDbFolder, folderPath, exclusionPatterns,
1271+
BackupFolderImpl(driver, database, dbPrefix, tmpDbFolder, folderPath, exclusionPatterns,
12551272
schemaOnly, useConsistentCopyTable, avoidCopy, preservePoolKinds, ordered, issues
12561273
);
12571274

‎ydb/public/lib/ydb_cli/dump/restore_impl.cpp

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -480,7 +480,7 @@ TRestoreResult TRestoreClient::RestoreFolder(
480480
}
481481

482482
if (IsFileExists(fsPath.Child(NFiles::CreateAsyncReplication().FileName))) {
483-
return RestoreReplication(fsPath, objectDbPath, settings, oldEntries.contains(objectDbPath));
483+
return RestoreReplication(fsPath, dbRestoreRoot, dbPathRelativeToRestoreRoot, settings, oldEntries.contains(objectDbPath));
484484
}
485485

486486
if (IsFileExists(fsPath.Child(NFiles::Empty().FileName))) {
@@ -505,7 +505,7 @@ TRestoreResult TRestoreClient::RestoreFolder(
505505
} else if (IsFileExists(child.Child(NFiles::CreateCoordinationNode().FileName))) {
506506
result = RestoreCoordinationNode(child, childDbPath, settings, oldEntries.contains(childDbPath));
507507
} else if (IsFileExists(child.Child(NFiles::CreateAsyncReplication().FileName))) {
508-
result = RestoreReplication(child, childDbPath, settings, oldEntries.contains(childDbPath));
508+
result = RestoreReplication(child, dbRestoreRoot, Join('/', dbPathRelativeToRestoreRoot, child.GetName()), settings, oldEntries.contains(childDbPath));
509509
} else if (child.IsDirectory()) {
510510
result = RestoreFolder(child, dbRestoreRoot, Join('/', dbPathRelativeToRestoreRoot, child.GetName()), settings, oldEntries);
511511
}
@@ -605,7 +605,8 @@ TRestoreResult TRestoreClient::RestoreTopic(
605605

606606
TRestoreResult TRestoreClient::RestoreReplication(
607607
const TFsPath& fsPath,
608-
const TString& dbPath,
608+
const TString& dbRestoreRoot,
609+
const TString& dbPathRelativeToRestoreRoot,
609610
const TRestoreSettings& settings,
610611
bool isAlreadyExisting)
611612
{
@@ -615,6 +616,7 @@ TRestoreResult TRestoreClient::RestoreReplication(
615616
return *error;
616617
}
617618

619+
const TString dbPath = dbRestoreRoot + dbPathRelativeToRestoreRoot;
618620
LOG_I("Restore async replication " << fsPath.GetPath().Quote() << " to " << dbPath.Quote());
619621

620622
if (settings.DryRun_) {
@@ -624,6 +626,9 @@ TRestoreResult TRestoreClient::RestoreReplication(
624626
auto query = ReadAsyncReplicationQuery(fsPath, Log.get());
625627

626628
NYql::TIssues issues;
629+
if (!RewriteObjectRefs(query, dbRestoreRoot, issues)) {
630+
return Result<TRestoreResult>(fsPath.GetPath(), EStatus::BAD_REQUEST, issues.ToString());
631+
}
627632
if (!RewriteCreateQuery(query, "CREATE ASYNC REPLICATION `{}`", dbPath, issues)) {
628633
return Result<TRestoreResult>(fsPath.GetPath(), EStatus::BAD_REQUEST, issues.ToString());
629634
}

‎ydb/public/lib/ydb_cli/dump/restore_impl.h

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ class TRestoreClient {
131131
TRestoreResult RestoreTable(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, bool isAlreadyExisting);
132132
TRestoreResult RestoreView(const TFsPath& fsPath, const TString& dbRestoreRoot, const TString& dbPathRelativeToRestoreRoot, const TRestoreSettings& settings, bool isAlreadyExisting);
133133
TRestoreResult RestoreTopic(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, bool isAlreadyExisting);
134-
TRestoreResult RestoreReplication(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, bool isAlreadyExisting);
134+
TRestoreResult RestoreReplication(const TFsPath& fsPath, const TString& dbRestoreRoot, const TString& dbPathRelativeToRestoreRoot, const TRestoreSettings& settings, bool isAlreadyExisting);
135135
TRestoreResult RestoreCoordinationNode(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, bool isAlreadyExisting);
136136
TRestoreResult RestoreDependentResources(const TFsPath& fsPath, const TString& dbPath);
137137
TRestoreResult RestoreRateLimiter(const TFsPath& fsPath, const TString& coordinationNodePath, const TString& resourcePath);

‎ydb/public/lib/ydb_cli/dump/util/query_utils.cpp

Lines changed: 49 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
#include <library/cpp/protobuf/util/simple_reflection.h>
1111

1212
#include <util/folder/pathsplit.h>
13+
#include <util/stream/str.h>
1314
#include <util/string/builder.h>
1415
#include <util/string/strip.h>
1516

@@ -47,13 +48,26 @@ TString RewriteAbsolutePath(TStringBuf path, TStringBuf backupRoot, TStringBuf r
4748
namespace {
4849

4950
struct TAbsolutePathRewriter {
51+
const TString Database;
5052
const TStringBuf BackupRoot;
5153
const TStringBuf RestoreRoot;
5254

55+
static TString BuildDatabaseToken(TStringBuf database) {
56+
if (database) {
57+
return TStringBuilder() << "`" << database << "/";
58+
} else {
59+
return "";
60+
}
61+
}
62+
5363
static bool IsAbsolutePath(TStringBuf path) {
5464
return path.StartsWith("`/") && path.EndsWith('`');
5565
}
5666

67+
bool IsInDatabase(TStringBuf path) const {
68+
return path.StartsWith(Database);
69+
}
70+
5771
TString RewriteAbsolutePath(TStringBuf path) const {
5872
if (BackupRoot == RestoreRoot) {
5973
return TString(path);
@@ -63,14 +77,15 @@ struct TAbsolutePathRewriter {
6377
}
6478

6579
public:
66-
explicit TAbsolutePathRewriter(TStringBuf backupRoot, TStringBuf restoreRoot)
67-
: BackupRoot(backupRoot)
80+
explicit TAbsolutePathRewriter(TStringBuf database, TStringBuf backupRoot, TStringBuf restoreRoot)
81+
: Database(BuildDatabaseToken(database))
82+
, BackupRoot(backupRoot)
6883
, RestoreRoot(restoreRoot)
6984
{
7085
}
7186

7287
TString operator()(const TString& path) const {
73-
if (IsAbsolutePath(path)) {
88+
if (IsAbsolutePath(path) && (!Database || IsInDatabase(path))) {
7489
return RewriteAbsolutePath(path);
7590
}
7691

@@ -175,8 +190,28 @@ struct TTableRefValidator {
175190
NYql::TIssues& Issues;
176191
};
177192

193+
TString GetOpt(TStringInput query, TStringBuf pattern) {
194+
TString line;
195+
while (query.ReadLine(line)) {
196+
StripInPlace(line);
197+
if (line.StartsWith(pattern)) {
198+
return TString(TStringBuf(line).Skip(pattern.size()).Chop(1 /* last " */));
199+
}
200+
}
201+
202+
return "";
203+
}
204+
178205
} // anonymous
179206

207+
TString GetBackupRoot(const TString& query) {
208+
return GetOpt(query, R"(-- backup root: ")");
209+
}
210+
211+
TString GetDatabase(const TString& query) {
212+
return GetOpt(query, R"(-- database: ")");
213+
}
214+
180215
bool SqlToProtoAst(const TString& queryStr, TRule_sql_query& queryProto, NYql::TIssues& issues) {
181216
NSQLTranslation::TTranslationSettings settings;
182217
if (!NSQLTranslation::ParseTranslationSettings(queryStr, settings, issues)) {
@@ -215,20 +250,20 @@ bool ValidateTableRefs(const TRule_sql_query& query, NYql::TIssues& issues) {
215250
}
216251

217252
template <typename TRef>
218-
TString RewriteRefs(const TRule_sql_query& query, TStringBuf backupRoot, TStringBuf restoreRoot) {
219-
TTokenCollector tokenCollector(TAbsolutePathRewriter(backupRoot, restoreRoot));
253+
TString RewriteRefs(const TRule_sql_query& query, TStringBuf db, TStringBuf backupRoot, TStringBuf restoreRoot) {
254+
TTokenCollector tokenCollector(TAbsolutePathRewriter(db, backupRoot, restoreRoot));
220255
VisitAllFields<TRef>(query, tokenCollector);
221256
return tokenCollector.Tokens;
222257
}
223258

224259
template <typename TRef>
225-
bool RewriteRefs(TString& queryStr, TStringBuf backupRoot, TStringBuf restoreRoot, NYql::TIssues& issues) {
260+
bool RewriteRefs(TString& queryStr, TStringBuf db, TStringBuf backupRoot, TStringBuf restoreRoot, NYql::TIssues& issues) {
226261
TRule_sql_query queryProto;
227262
if (!SqlToProtoAst(queryStr, queryProto, issues)) {
228263
return false;
229264
}
230265

231-
const auto rewrittenQuery = RewriteRefs<TRef>(queryProto, backupRoot, restoreRoot);
266+
const auto rewrittenQuery = RewriteRefs<TRef>(queryProto, db, backupRoot, restoreRoot);
232267
// formatting here is necessary for the view to have pretty text inside it after the creation
233268
if (!Format(rewrittenQuery, queryStr, issues)) {
234269
return false;
@@ -238,11 +273,15 @@ bool RewriteRefs(TString& queryStr, TStringBuf backupRoot, TStringBuf restoreRoo
238273
}
239274

240275
bool RewriteTableRefs(TString& query, TStringBuf backupRoot, TStringBuf restoreRoot, NYql::TIssues& issues) {
241-
return RewriteRefs<TRule_table_ref>(query, backupRoot, restoreRoot, issues);
276+
return RewriteRefs<TRule_table_ref>(query, "", backupRoot, restoreRoot, issues);
277+
}
278+
279+
bool RewriteTableRefs(TString& query, TStringBuf restoreRoot, NYql::TIssues& issues) {
280+
return RewriteRefs<TRule_table_ref>(query, GetDatabase(query), GetBackupRoot(query), restoreRoot, issues);
242281
}
243282

244-
bool RewriteObjectRefs(TString& query, TStringBuf backupRoot, TStringBuf restoreRoot, NYql::TIssues& issues) {
245-
return RewriteRefs<TRule_object_ref>(query, backupRoot, restoreRoot, issues);
283+
bool RewriteObjectRefs(TString& query, TStringBuf restoreRoot, NYql::TIssues& issues) {
284+
return RewriteRefs<TRule_object_ref>(query, GetDatabase(query), GetBackupRoot(query), restoreRoot, issues);
246285
}
247286

248287
bool RewriteCreateQuery(TString& query, std::string_view pattern, const std::string& dbPath, NYql::TIssues& issues) {
@@ -255,19 +294,4 @@ bool RewriteCreateQuery(TString& query, std::string_view pattern, const std::str
255294
return false;
256295
}
257296

258-
TString GetBackupRoot(TStringInput query) {
259-
constexpr TStringBuf targetLinePrefix = "-- backup root: \"";
260-
constexpr TStringBuf discardedSuffix = "\"";
261-
262-
TString line;
263-
while (query.ReadLine(line)) {
264-
StripInPlace(line);
265-
if (line.StartsWith(targetLinePrefix)) {
266-
return TString(TStringBuf(line).Skip(targetLinePrefix.size()).Chop(discardedSuffix.size()));
267-
}
268-
}
269-
270-
return "";
271-
}
272-
273297
} // NYdb::NDump

‎ydb/public/lib/ydb_cli/dump/util/query_utils.h

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
#pragma once
22

33
#include <util/generic/string.h>
4-
#include <util/stream/str.h>
54

65
#include <string>
76
#include <string_view>
@@ -23,9 +22,11 @@ bool ValidateTableRefs(const NSQLv1Generated::TRule_sql_query& query, NYql::TIss
2322

2423
TString RewriteAbsolutePath(TStringBuf path, TStringBuf backupRoot, TStringBuf restoreRoot);
2524
bool RewriteTableRefs(TString& query, TStringBuf backupRoot, TStringBuf restoreRoot, NYql::TIssues& issues);
26-
bool RewriteObjectRefs(TString& query, TStringBuf backupRoot, TStringBuf restoreRoot, NYql::TIssues& issues);
25+
bool RewriteTableRefs(TString& query, TStringBuf restoreRoot, NYql::TIssues& issues);
26+
bool RewriteObjectRefs(TString& query, TStringBuf restoreRoot, NYql::TIssues& issues);
2727
bool RewriteCreateQuery(TString& query, std::string_view pattern, const std::string& dbPath, NYql::TIssues& issues);
2828

29-
TString GetBackupRoot(TStringInput query);
29+
TString GetBackupRoot(const TString& query);
30+
TString GetDatabase(const TString& query);
3031

3132
}

0 commit comments

Comments
 (0)