Skip to content

Commit 7059c21

Browse files
pixccpixcc
authored
Add permissions to ydb tools restore (#7247)
1 parent 0820b69 commit 7059c21

File tree

13 files changed

+598
-53
lines changed

13 files changed

+598
-53
lines changed

‎ydb/library/backup/backup.cpp

Lines changed: 6 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,10 @@
55
#include "util.h"
66

77
#include <ydb/public/api/protos/ydb_table.pb.h>
8+
#include <ydb/public/lib/ydb_cli/common/recursive_remove.h>
9+
#include <ydb/public/lib/ydb_cli/dump/util/util.h>
810
#include <ydb/public/lib/yson_value/ydb_yson_value.h>
911
#include <ydb/public/sdk/cpp/client/ydb_table/table.h>
10-
#include <ydb/public/lib/ydb_cli/common/recursive_remove.h>
1112

1213
#include <library/cpp/containers/stack_vector/stack_vec.h>
1314
#include <library/cpp/string_utils/quote/quote.h>
@@ -432,8 +433,8 @@ Ydb::Table::CreateTableRequest ProtoFromTableDescription(const NTable::TTableDes
432433

433434
NScheme::TSchemeEntry DescribePath(TDriver driver, const TString& fullPath) {
434435
NScheme::TSchemeClient client(driver);
435-
436-
auto status = client.DescribePath(fullPath).GetValueSync();
436+
437+
auto status = NDump::DescribePath(client, fullPath);
437438
VerifyStatus(status);
438439
LOG_DEBUG("Path is described, fullPath: " << fullPath);
439440

@@ -596,9 +597,6 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba
596597
BackupTable(driver, dbIt.GetTraverseRoot(), backupPrefix, dbIt.GetRelPath(),
597598
childFolderPath, schemaOnly, preservePoolKinds, ordered);
598599
childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists();
599-
} else if (dbIt.IsDir()) {
600-
BackupPermissions(driver, dbIt.GetTraverseRoot(), dbIt.GetRelPath(), childFolderPath);
601-
childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists();
602600
}
603601
} else if (!avoidCopy) {
604602
if (dbIt.IsTable()) {
@@ -632,6 +630,7 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba
632630
Y_ENSURE(!childFolderPath.Child(INCOMPLETE_FILE_NAME).Exists());
633631
} else if (dbIt.IsDir()) {
634632
MaybeCreateEmptyFile(childFolderPath);
633+
BackupPermissions(driver, dbIt.GetTraverseRoot(), dbIt.GetRelPath(), childFolderPath);
635634
}
636635

637636
childFolderPath.Child(INCOMPLETE_FILE_NAME).DeleteIfExists();
@@ -670,8 +669,8 @@ void BackupFolderImpl(TDriver driver, const TString& dbPrefix, const TString& ba
670669
DropTable(driver, tmpTablePath);
671670
}
672671
} else if (dbIt.IsDir()) {
673-
BackupPermissions(driver, dbIt.GetTraverseRoot(), dbIt.GetRelPath(), childFolderPath);
674672
MaybeCreateEmptyFile(childFolderPath);
673+
BackupPermissions(driver, dbIt.GetTraverseRoot(), dbIt.GetRelPath(), childFolderPath);
675674
if (!avoidCopy) {
676675
RemoveClusterDirectory(driver, tmpTablePath);
677676
}

‎ydb/library/backup/ya.make

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ PEERDIR(
99
ydb/public/api/grpc
1010
ydb/public/api/protos
1111
ydb/public/lib/ydb_cli/common
12+
ydb/public/lib/ydb_cli/dump/util
1213
ydb/public/lib/yson_value
1314
ydb/public/sdk/cpp/client/ydb_proto
1415
ydb/public/sdk/cpp/client/ydb_scheme

‎ydb/public/lib/ydb_cli/commands/ydb_tools.cpp

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,7 +54,7 @@ void TCommandDump::Config(TConfig& config) {
5454
config.Opts->AddLongOption('o', "output", "[Required] Path in a local filesystem to a directory to place dump into."
5555
" Directory should either not exist or be empty.")
5656
.StoreResult(&FilePath);
57-
config.Opts->AddLongOption("scheme-only", "Dump only scheme")
57+
config.Opts->AddLongOption("scheme-only", "Dump only scheme including ACL and owner")
5858
.StoreTrue(&IsSchemeOnly);
5959
config.Opts->AddLongOption("avoid-copy", "Avoid copying."
6060
" By default, YDB makes a copy of a table before dumping it to reduce impact on workload and ensure consistency.\n"
@@ -142,6 +142,9 @@ void TCommandRestore::Config(TConfig& config) {
142142

143143
config.Opts->AddLongOption("restore-indexes", "Whether to restore indexes or not")
144144
.DefaultValue(defaults.RestoreIndexes_).StoreResult(&RestoreIndexes);
145+
146+
config.Opts->AddLongOption("restore-acl", "Whether to restore ACL and owner or not")
147+
.DefaultValue(defaults.RestoreACL_).StoreResult(&RestoreACL);
145148

146149
config.Opts->AddLongOption("skip-document-tables", TStringBuilder()
147150
<< "Document API tables cannot be restored for now. "
@@ -197,6 +200,7 @@ int TCommandRestore::Run(TConfig& config) {
197200
.DryRun(IsDryRun)
198201
.RestoreData(RestoreData)
199202
.RestoreIndexes(RestoreIndexes)
203+
.RestoreACL(RestoreACL)
200204
.SkipDocumentTables(SkipDocumentTables)
201205
.SavePartialResult(SavePartialResult)
202206
.RowsPerRequest(NYdb::SizeFromString(RowsPerRequest))

‎ydb/public/lib/ydb_cli/commands/ydb_tools.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ class TCommandRestore : public TToolsCommand, public TCommandWithPath {
5858
bool IsDryRun = false;
5959
bool RestoreData = true;
6060
bool RestoreIndexes = true;
61+
bool RestoreACL = true;
6162
bool SkipDocumentTables = false;
6263
bool SavePartialResult = false;
6364
TString UploadBandwidth;

‎ydb/public/lib/ydb_cli/dump/dump.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ namespace NYdb {
1313
namespace NDump {
1414

1515
extern const char SCHEME_FILE_NAME[] = "scheme.pb";
16+
extern const char PERMISSIONS_FILE_NAME[] = "permissions.pb";
1617
extern const char INCOMPLETE_FILE_NAME[] = "incomplete";
1718
extern const char EMPTY_FILE_NAME[] = "empty_dir";
1819

‎ydb/public/lib/ydb_cli/dump/dump.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ namespace NYdb {
1010
namespace NDump {
1111

1212
extern const char SCHEME_FILE_NAME[10];
13+
extern const char PERMISSIONS_FILE_NAME[15];
1314
extern const char INCOMPLETE_FILE_NAME[11];
1415
extern const char EMPTY_FILE_NAME[10];
1516

@@ -64,6 +65,7 @@ struct TRestoreSettings: public TOperationRequestSettings<TRestoreSettings> {
6465
FLUENT_SETTING_DEFAULT(bool, DryRun, false);
6566
FLUENT_SETTING_DEFAULT(bool, RestoreData, true);
6667
FLUENT_SETTING_DEFAULT(bool, RestoreIndexes, true);
68+
FLUENT_SETTING_DEFAULT(bool, RestoreACL, true);
6769
FLUENT_SETTING_DEFAULT(bool, SkipDocumentTables, false);
6870
FLUENT_SETTING_DEFAULT(bool, SavePartialResult, false);
6971

‎ydb/public/lib/ydb_cli/dump/restore_impl.cpp

Lines changed: 72 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44

55
#include <ydb/public/api/protos/ydb_table.pb.h>
66
#include <ydb/public/sdk/cpp/client/ydb_proto/accessor.h>
7+
#include <ydb/public/lib/ydb_cli/common/recursive_list.h>
78
#include <ydb/public/lib/ydb_cli/common/recursive_remove.h>
89
#include <ydb/public/lib/ydb_cli/common/retry_func.h>
910
#include <ydb/public/lib/ydb_cli/dump/util/util.h>
@@ -19,6 +20,7 @@
1920
namespace NYdb {
2021
namespace NDump {
2122

23+
using namespace NConsoleClient;
2224
using namespace NImport;
2325
using namespace NOperation;
2426
using namespace NScheme;
@@ -43,6 +45,12 @@ TTableDescription TableDescriptionFromProto(const Ydb::Table::CreateTableRequest
4345
return TProtoAccessor::FromProto(proto);
4446
}
4547

48+
Ydb::Scheme::ModifyPermissionsRequest ReadPermissions(const TString& fsPath) {
49+
Ydb::Scheme::ModifyPermissionsRequest proto;
50+
Y_ENSURE(google::protobuf::TextFormat::ParseFromString(TFileInput(fsPath).ReadAll(), &proto));
51+
return proto;
52+
}
53+
4654
bool HasRunningIndexBuilds(TOperationClient& client, const TString& dbPath) {
4755
const ui64 pageSize = 100;
4856
TString pageToken;
@@ -131,34 +139,34 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP
131139
dbBasePath = dbBasePath.Parent();
132140
}
133141

134-
auto oldDirectoryList = SchemeClient.ListDirectory(dbBasePath).GetValueSync();
135-
if (!oldDirectoryList.IsSuccess()) {
142+
auto oldDirectoryList = RecursiveList(SchemeClient, dbBasePath);
143+
if (!oldDirectoryList.Status.IsSuccess()) {
136144
return Result<TRestoreResult>(EStatus::SCHEME_ERROR, "Can not list existing directory");
137145
}
138146

139147
THashSet<TString> oldEntries;
140-
for (const auto& entry : oldDirectoryList.GetChildren()) {
148+
for (const auto& entry : oldDirectoryList.Entries) {
141149
oldEntries.insert(entry.Name);
142150
}
143151

144152
// restore
145-
auto restoreResult = RestoreFolder(fsPath, dbPath, settings);
153+
auto restoreResult = RestoreFolder(fsPath, dbPath, settings, oldEntries);
146154
if (restoreResult.IsSuccess() || settings.SavePartialResult_) {
147155
return restoreResult;
148156
}
149157

150158
// cleanup
151-
auto newDirectoryList = SchemeClient.ListDirectory(dbBasePath).GetValueSync();
152-
if (!newDirectoryList.IsSuccess()) {
159+
auto newDirectoryList = RecursiveList(SchemeClient, dbBasePath);
160+
if (!newDirectoryList.Status.IsSuccess()) {
153161
return restoreResult;
154162
}
155163

156-
for (const auto& entry : newDirectoryList.GetChildren()) {
164+
for (const auto& entry : newDirectoryList.Entries) {
157165
if (oldEntries.contains(entry.Name)) {
158166
continue;
159167
}
160168

161-
auto fullPath = dbBasePath.Child(entry.Name);
169+
const auto& fullPath = entry.Name; // RecursiveList returns full path instead of entry's name
162170

163171
switch (entry.Type) {
164172
case ESchemeEntryType::Directory: {
@@ -185,7 +193,9 @@ TRestoreResult TRestoreClient::Restore(const TString& fsPath, const TString& dbP
185193
return restoreResult;
186194
}
187195

188-
TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings) {
196+
TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TString& dbPath,
197+
const TRestoreSettings& settings, const THashSet<TString>& oldEntries)
198+
{
189199
if (!fsPath) {
190200
return Result<TRestoreResult>(EStatus::BAD_REQUEST, "Folder is not specified");
191201
}
@@ -206,11 +216,11 @@ TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TStrin
206216
}
207217

208218
if (IsFileExists(fsPath.Child(SCHEME_FILE_NAME))) {
209-
return RestoreTable(fsPath, Join('/', dbPath, fsPath.GetName()), settings);
219+
return RestoreTable(fsPath, Join('/', dbPath, fsPath.GetName()), settings, oldEntries);
210220
}
211221

212222
if (IsFileExists(fsPath.Child(EMPTY_FILE_NAME))) {
213-
return MakeDirectory(SchemeClient, Join('/', dbPath, fsPath.GetName()));
223+
return RestoreEmptyDir(fsPath, Join('/', dbPath, fsPath.GetName()), settings, oldEntries);
214224
}
215225

216226
TMaybe<TRestoreResult> result;
@@ -219,26 +229,24 @@ TRestoreResult TRestoreClient::RestoreFolder(const TFsPath& fsPath, const TStrin
219229
fsPath.List(children);
220230
for (const auto& child : children) {
221231
if (IsFileExists(child.Child(SCHEME_FILE_NAME))) {
222-
result = RestoreTable(child, Join('/', dbPath, child.GetName()), settings);
232+
result = RestoreTable(child, Join('/', dbPath, child.GetName()), settings, oldEntries);
223233
} else if (IsFileExists(child.Child(EMPTY_FILE_NAME))) {
224-
result = MakeDirectory(SchemeClient, Join('/', dbPath, child.GetName()));
225-
} else {
226-
result = RestoreFolder(child, Join('/', dbPath, child.GetName()), settings);
234+
result = RestoreEmptyDir(child, Join('/', dbPath, child.GetName()), settings, oldEntries);
235+
} else if (child.IsDirectory()) {
236+
result = RestoreFolder(child, Join('/', dbPath, child.GetName()), settings, oldEntries);
227237
}
228238

229-
if (!result->IsSuccess()) {
239+
if (result.Defined() && !result->IsSuccess()) {
230240
return *result;
231241
}
232242
}
233-
234-
if (!result) {
235-
return Result<TRestoreResult>();
236-
}
237-
238-
return *result;
243+
244+
return RestorePermissions(fsPath, dbPath, settings, oldEntries);
239245
}
240246

241-
TRestoreResult TRestoreClient::RestoreTable(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings) {
247+
TRestoreResult TRestoreClient::RestoreTable(const TFsPath& fsPath, const TString& dbPath,
248+
const TRestoreSettings& settings, const THashSet<TString>& oldEntries)
249+
{
242250
if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) {
243251
return Result<TRestoreResult>(EStatus::BAD_REQUEST,
244252
TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
@@ -268,7 +276,7 @@ TRestoreResult TRestoreClient::RestoreTable(const TFsPath& fsPath, const TString
268276
}
269277
}
270278

271-
return Result<TRestoreResult>();
279+
return RestorePermissions(fsPath, dbPath, settings, oldEntries);
272280
}
273281

274282
TRestoreResult TRestoreClient::CheckSchema(const TString& dbPath, const TTableDescription& desc) {
@@ -444,5 +452,45 @@ TRestoreResult TRestoreClient::RestoreIndexes(const TString& dbPath, const TTabl
444452
return Result<TRestoreResult>();
445453
}
446454

455+
TRestoreResult TRestoreClient::RestorePermissions(const TFsPath& fsPath, const TString& dbPath,
456+
const TRestoreSettings& settings, const THashSet<TString>& oldEntries)
457+
{
458+
if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) {
459+
return Result<TRestoreResult>(EStatus::BAD_REQUEST,
460+
TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
461+
}
462+
463+
if (!settings.RestoreACL_) {
464+
return Result<TRestoreResult>();
465+
}
466+
467+
if (oldEntries.contains(dbPath)) {
468+
return Result<TRestoreResult>();
469+
}
470+
471+
if (!fsPath.Child(PERMISSIONS_FILE_NAME).Exists()) {
472+
return Result<TRestoreResult>();
473+
}
474+
475+
auto permissions = ReadPermissions(fsPath.Child(PERMISSIONS_FILE_NAME));
476+
return ModifyPermissions(SchemeClient, dbPath, TModifyPermissionsSettings(permissions));
477+
}
478+
479+
TRestoreResult TRestoreClient::RestoreEmptyDir(const TFsPath& fsPath, const TString &dbPath,
480+
const TRestoreSettings& settings, const THashSet<TString>& oldEntries)
481+
{
482+
if (fsPath.Child(INCOMPLETE_FILE_NAME).Exists()) {
483+
return Result<TRestoreResult>(EStatus::BAD_REQUEST,
484+
TStringBuilder() << "There is incomplete file in folder: " << fsPath.GetPath());
485+
}
486+
487+
auto result = MakeDirectory(SchemeClient, dbPath);
488+
if (!result.IsSuccess()) {
489+
return result;
490+
}
491+
492+
return RestorePermissions(fsPath, dbPath, settings, oldEntries);
493+
}
494+
447495
} // NDump
448496
} // NYdb

‎ydb/public/lib/ydb_cli/dump/restore_impl.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,12 +36,14 @@ class IDataWriter {
3636
} // NPrivate
3737

3838
class TRestoreClient {
39-
TRestoreResult RestoreFolder(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings);
40-
TRestoreResult RestoreTable(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings);
39+
TRestoreResult RestoreFolder(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const THashSet<TString>& oldEntries);
40+
TRestoreResult RestoreEmptyDir(const TFsPath& fsPath, const TString &dbPath, const TRestoreSettings& settings, const THashSet<TString>& oldEntries);
41+
TRestoreResult RestoreTable(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const THashSet<TString>& oldEntries);
4142

4243
TRestoreResult CheckSchema(const TString& dbPath, const NTable::TTableDescription& desc);
4344
TRestoreResult RestoreData(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const NTable::TTableDescription& desc);
4445
TRestoreResult RestoreIndexes(const TString& dbPath, const NTable::TTableDescription& desc);
46+
TRestoreResult RestorePermissions(const TFsPath& fsPath, const TString& dbPath, const TRestoreSettings& settings, const THashSet<TString>& oldEntries);
4547

4648
public:
4749
explicit TRestoreClient(

‎ydb/public/lib/ydb_cli/dump/util/util.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,10 @@ TStatus MakeDirectory(TSchemeClient& schemeClient, const TString& path, const TM
3434
});
3535
}
3636

37+
TStatus ModifyPermissions(TSchemeClient& schemeClient, const TString& path, const TModifyPermissionsSettings& settings) {
38+
return NConsoleClient::RetryFunction([&]() -> TStatus {
39+
return schemeClient.ModifyPermissions(path, settings).ExtractValueSync();
40+
});
41+
}
42+
3743
}

‎ydb/public/lib/ydb_cli/dump/util/util.h

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,4 +51,8 @@ TStatus MakeDirectory(
5151
const TString& path,
5252
const NScheme::TMakeDirectorySettings& settings = {});
5353

54+
TStatus ModifyPermissions(
55+
NScheme::TSchemeClient& schemeClient,
56+
const TString& path,
57+
const NScheme::TModifyPermissionsSettings& settings = {});
5458
}

0 commit comments

Comments
 (0)