Make permissions required in import if they were exported (#19331)

Author: UgnineSirdis

ydb/core/backup/common/metadata.cpp

@@ -34,11 +34,26 @@ const std::optional<std::vector<TChangefeedMetadata>>& TMetadata::GetChangefeeds
     return Changefeeds;
 }
 
+void TMetadata::SetEnablePermissions(bool enablePermissions) {
+    EnablePermissions = enablePermissions;
+}
+
+bool TMetadata::HasEnablePermissions() const {
+    return EnablePermissions.has_value();
+}
+
+bool TMetadata::GetEnablePermissions() const {
+    return *EnablePermissions;
+}
+
 TString TMetadata::Serialize() const {
     NJson::TJsonMap m;
     if (Version.Defined()) {
         m["version"] = *Version;
     }
+    if (EnablePermissions) {
+        m["permissions"] = static_cast<int>(*EnablePermissions);
+    }
 
     NJson::TJsonArray fullBackups;
     for (const auto& [tp, b] : FullBackups) {
@@ -76,6 +91,11 @@ TMetadata TMetadata::Deserialize(const TString& metadata) {
         result.Version = value.GetUIntegerSafe();
     }
 
+    if (json.Has("permissions")) {
+        const int val = json["permissions"].GetInteger();
+        result.EnablePermissions = val != 0;
+    }
+
     if (json.Has("changefeeds")) {
         // Changefeeds can be absent in older versions of metadata
         result.Changefeeds.emplace(); // explicitly say that the listing of changefeeds is throgh metadata

ydb/core/backup/common/metadata.h

@@ -55,14 +55,30 @@ class TMetadata {
     void AddChangefeed(const TChangefeedMetadata& changefeed);
     const std::optional<std::vector<TChangefeedMetadata>>& GetChangefeeds() const;
 
+    void SetEnablePermissions(bool enablePermissions = true);
+    bool HasEnablePermissions() const;
+    bool GetEnablePermissions() const;
+
     TString Serialize() const;
     static TMetadata Deserialize(const TString& metadata);
+
 private:
     TString ConsistencyKey;
     TMap<TVirtualTimestamp, TFullBackupMetadata::TPtr> FullBackups;
     TMap<TVirtualTimestamp, TLogMetadata::TPtr> Logs;
-    std::optional<std::vector<TChangefeedMetadata>> Changefeeds;
     TMaybeFail<ui64> Version;
+
+    // Changefeeds:
+    // Undefined (previous versions): we don't know if we see the export with changefeeds or without them, so list suitable S3 files to find out all changefeeds
+    // []: The export has no changefeeds
+    // [...]: The export must have all changefeeds listed here
+    std::optional<std::vector<TChangefeedMetadata>> Changefeeds;
+
+    // EnablePermissions:
+    // Undefined (previous versions): we don't know if we see the export with permissions or without them, so check S3 for the permissions file existence
+    // 0: The export has no permissions
+    // 1: The export must have permissions file
+    std::optional<bool> EnablePermissions;
 };
 
 TString NormalizeItemPath(const TString& path);

ydb/core/tx/datashard/export_s3_uploader.cpp

@@ -946,6 +946,7 @@ IActor* TS3Export::CreateUploader(const TActorId& dataShard, ui64 txId) const {
 
     TMetadata metadata;
     metadata.SetVersion(Task.GetEnableChecksums() ? 1 : 0);
+    metadata.SetEnablePermissions(Task.GetEnablePermissions());
 
     TVector<TChangefeedExportDescriptions> changefeeds;
     if (AppData()->FeatureFlags.GetEnableChangefeedsExport()) {

ydb/core/tx/schemeshard/schemeshard_export__create.cpp

@@ -387,6 +387,7 @@ struct TSchemeShard::TExport::TTxProgress: public TSchemeShard::TXxport::TTxBase
             // to do: enable view checksum validation
             constexpr bool EnableChecksums = false;
             metadata.SetVersion(EnableChecksums ? 1 : 0);
+            metadata.SetEnablePermissions(exportInfo.EnablePermissions);
 
             TMaybe<NBackup::TEncryptionIV> iv;
             if (exportSettings.has_encryption_settings()) {

ydb/core/tx/schemeshard/schemeshard_import_getters.cpp

@@ -373,7 +373,13 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
             << ", result# " << result);
 
         if (NoObjectFound(result.GetError().GetErrorType())) {
-            StartDownloadingChangefeeds(); // permissions are optional
+            Y_ABORT_UNLESS(ItemIdx < ImportInfo->Items.size());
+            auto& item = ImportInfo->Items.at(ItemIdx);
+            if (!item.Metadata.HasEnablePermissions()) {
+                StartDownloadingChangefeeds(); // permissions are optional if we don't know if they were created during export
+            } else {
+                return Reply(Ydb::StatusIds::BAD_REQUEST, "No permissions file found");
+            }
             return;
         } else if (!CheckResult(result, "HeadObject")) {
             return;
@@ -441,6 +447,9 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
         if (item.Metadata.HasVersion() && item.Metadata.GetVersion() == 0) {
             NeedValidateChecksums = false;
         }
+        if (item.Metadata.HasEnablePermissions() && !item.Metadata.GetEnablePermissions()) {
+            NeedDownloadPermissions = false;
+        }
 
         auto nextStep = [this]() {
             StartDownloadingScheme();
@@ -829,8 +838,7 @@ class TSchemeGetter: public TGetterFromS3<TSchemeGetter> {
     TVector<TString> ChangefeedsPrefixes;
     ui64 IndexDownloadedChangefeed = 0;
 
-    const bool NeedDownloadPermissions = true;
-
+    bool NeedDownloadPermissions = true;
     bool NeedValidateChecksums = true;
 }; // TSchemeGetter
 

ydb/core/tx/schemeshard/ut_export/ut_export.cpp

@@ -2440,7 +2440,7 @@ partitioning_settings {
 
         const auto* metadataChecksum = S3Mock().GetData().FindPtr("/metadata.json.sha256");
         UNIT_ASSERT(metadataChecksum);
-        UNIT_ASSERT_VALUES_EQUAL(*metadataChecksum, "843d475f20c5e337fdd3ef4e83b63081c182e26791cd891f7739be8b53167b1b metadata.json");
+        UNIT_ASSERT_VALUES_EQUAL(*metadataChecksum, "29c79eb8109b4142731fc894869185d6c0e99c4b2f605ea3fc726b0328b8e316 metadata.json");
 
         const auto* schemeChecksum = S3Mock().GetData().FindPtr("/scheme.pb.sha256");
         UNIT_ASSERT(schemeChecksum);
@@ -2507,7 +2507,7 @@ partitioning_settings {
 
         const auto* metadataChecksum = S3Mock().GetData().FindPtr("/metadata.json.sha256");
         UNIT_ASSERT(metadataChecksum);
-        UNIT_ASSERT_VALUES_EQUAL(*metadataChecksum, "843d475f20c5e337fdd3ef4e83b63081c182e26791cd891f7739be8b53167b1b metadata.json");
+        UNIT_ASSERT_VALUES_EQUAL(*metadataChecksum, "fbb85825fb12c5f38661864db884ba3fd1512fc4b0a2a41960d7d62d19318ab6 metadata.json");
 
         const auto* schemeChecksum = S3Mock().GetData().FindPtr("/scheme.pb.sha256");
         UNIT_ASSERT(schemeChecksum);