added GetParameterTypes() to TDataQuery (#16095)

Author: asmyasnikov

include/ydb-cpp-sdk/client/table/table.h

@@ -1955,6 +1955,7 @@ class TDataQuery {
     const std::string& GetId() const;
     const std::optional<std::string>& GetText() const;
     TParamsBuilder GetParamsBuilder() const;
+    std::map<std::string, TType> GetParameterTypes() const;
 
     TAsyncDataQueryResult Execute(const TTxControl& txControl,
         const TExecDataQuerySettings& settings = TExecDataQuerySettings());

src/api/protos/ydb_export.proto

@@ -1,6 +1,7 @@
 syntax = "proto3";
 option cc_enable_arenas = true;
 
+import "src/api/protos/annotations/sensitive.proto";
 import "src/api/protos/annotations/validation.proto";
 import "src/api/protos/ydb_operation.proto";
 
@@ -86,23 +87,27 @@ message ExportToS3Settings {
     };
 
     message Item {
-        // Database path to a table to be exported
+        // Database path to a table/directory to be exported
         string source_path = 1 [(required) = true];
 
         /* Tables are exported to one or more S3 objects.
            The object name begins with 'destination_prefix'.
            This prefix will be followed by '/data_PartNumber', where 'PartNumber'
            represents the index of the part, starting at zero.
+           Not required if the default `destination_prefix` is set.
+           If not specified, actual S3 path is the default `destination_prefix` concatenated with:
+            * The object path relative to the global `source_path` for a non-encrypted export
+            * The anonymized path for an encrypted export
         */
-        string destination_prefix = 2 [(required) = true];
+        string destination_prefix = 2;
     }
 
     string endpoint = 1 [(required) = true];
     Scheme scheme = 2; // HTTPS if not specified
     string bucket = 3 [(required) = true];
     string access_key = 4 [(required) = true];
     string secret_key = 5 [(required) = true];
-    repeated Item items = 6 [(size).ge = 1];
+    repeated Item items = 6;
     string description = 7 [(length).le = 128];
     uint32 number_of_retries = 8;
     StorageClass storage_class = 9;
@@ -120,6 +125,20 @@ message ExportToS3Settings {
     // details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
     // it is especially useful for custom s3 implementations
     bool disable_virtual_addressing = 12;
+
+    // Database root if not provided.
+    // All object names are calculated and written relative to this path.
+    string source_path = 13;
+
+    // A default S3 path prefix for all export items.
+    // When specified, export writes SchemaMapping file with the list of objects.
+    // Must be provided for an encrypted backup.
+    string destination_prefix = 14;
+
+    // Settings for data encryption.
+    // If encryption_settings field is not specified,
+    // the resulting data will not be encrypted.
+    EncryptionSettings encryption_settings = 15;
 }
 
 message ExportToS3Result {
@@ -141,3 +160,24 @@ message ExportToS3Response {
     // operation.metadata = ExportToS3Metadata
     Ydb.Operations.Operation operation = 1;
 }
+
+// Export encryption settings
+// Don't specify this struct for unencrypted exports
+message EncryptionSettings {
+    // Algorithm for export encryption.
+    // Not required in case of import/list operation.
+    // Currently the following algorithms are supported:
+    //  AES-128-GCM
+    //  AES-256-GCM
+    //  ChaCha20-Poly1305
+    string encryption_algorithm = 1;
+
+    oneof Key {
+        SymmetricKey symmetric_key = 2;
+    }
+
+    message SymmetricKey {
+        // This key will be used for data encryption
+        bytes key = 1 [(Ydb.sensitive) = true];
+    }
+}

src/api/protos/ydb_import.proto

@@ -2,6 +2,7 @@ syntax = "proto3";
 option cc_enable_arenas = true;
 
 import "src/api/protos/annotations/validation.proto";
+import "src/api/protos/ydb_export.proto";
 import "src/api/protos/ydb_operation.proto";
 
 import "google/protobuf/timestamp.proto";
@@ -39,25 +40,33 @@ message ImportFromS3Settings {
     }
 
     message Item {
-        /* YDB tables in S3 are stored in one or more objects (see ydb_export.proto).
-           The object name begins with 'source_prefix'.
-           This prefix is followed by:
-             * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
-             * '/scheme.pb' - object with information about scheme, indexes, etc;
-             * '/permissions.pb' - object with information about ACL and owner.
-        */
-        string source_prefix = 1 [(required) = true];
-
-        // Database path to a table to import to.
-        string destination_path = 2 [(required) = true];
+        oneof Source {
+            /* YDB database objects in S3 are stored in one or more S3 objects (see ydb_export.proto).
+            The S3 object name begins with a prefix, followed by:
+                * '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
+                * '/scheme.pb' - object with information about scheme, indexes, etc;
+                * '/permissions.pb' - object with information about ACL and owner.
+            */
+
+            // The S3 object prefix can be either provided explicitly
+            string source_prefix = 1;
+
+            // Or, if the export contains the database objects list, you may specify the database object name, and the S3 prefix will be looked up in the database objects list by the import procedure
+            string source_path = 3;
+        }
+
+        // Database path to a database object to import the item to
+        // Resolved relative to the default destination_path
+        // May be omitted if the item's source_path is specified, in this case will be taken equal to it
+        string destination_path = 2;
     }
 
     string endpoint = 1 [(required) = true];
     Scheme scheme = 2; // HTTPS if not specified
     string bucket = 3 [(required) = true];
     string access_key = 4 [(required) = true];
     string secret_key = 5 [(required) = true];
-    repeated Item items = 6 [(size).ge = 1];
+    repeated Item items = 6; // Empty collection means import of all export objects
     string description = 7 [(length).le = 128];
     uint32 number_of_retries = 8;
 
@@ -76,6 +85,20 @@ message ImportFromS3Settings {
 
     // Skip checksum validation during import
     bool skip_checksum_validation = 12;
+
+    // A default path prefix for all items,
+    // determines that the import works with the list of objects in the SchemaMapping file.
+    // Must be provided to import an encrypted export.
+    string source_prefix = 13;
+
+    // Destination path to restore paths inside database
+    // Default value is database root
+    string destination_path = 14;
+
+    // Settings how data is encrypted.
+    // If encryption_settings field is not specified,
+    // the resulting data is considered not encrypted.
+    Ydb.Export.EncryptionSettings encryption_settings = 15;
 }
 
 message ImportFromS3Result {

src/api/protos/ydb_scheme.proto

@@ -66,6 +66,7 @@ message Entry {
         EXTERNAL_DATA_SOURCE = 19;
         VIEW = 20;
         RESOURCE_POOL = 21;
+        TRANSFER = 23;
     }
 
     // Name of scheme entry (dir2 of /dir1/dir2)

src/client/table/table.cpp

@@ -2040,6 +2040,14 @@ const std::optional<std::string>& TDataQuery::GetText() const {
     return Impl_->GetText();
 }
 
+std::map<std::string, TType> TDataQuery::GetParameterTypes() const {
+    std::map<std::string, TType> typesMap;
+    for (const auto& param : Impl_->ParameterTypes_) {
+        typesMap.emplace(param.first, TType(param.second));
+    }
+    return typesMap;
+}
+
 TParamsBuilder TDataQuery::GetParamsBuilder() const {
     return TParamsBuilder(Impl_->ParameterTypes_);
 }