Skip to content

Commit 9892985

Browse files
asmyasnikovGazizonoki
authored andcommitted
added GetParameterTypes() to TDataQuery (#16095)
1 parent cc3c9e3 commit 9892985

File tree

5 files changed

+88
-15
lines changed

5 files changed

+88
-15
lines changed

include/ydb-cpp-sdk/client/table/table.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1955,6 +1955,7 @@ class TDataQuery {
19551955
const std::string& GetId() const;
19561956
const std::optional<std::string>& GetText() const;
19571957
TParamsBuilder GetParamsBuilder() const;
1958+
std::map<std::string, TType> GetParameterTypes() const;
19581959

19591960
TAsyncDataQueryResult Execute(const TTxControl& txControl,
19601961
const TExecDataQuerySettings& settings = TExecDataQuerySettings());

src/api/protos/ydb_export.proto

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
syntax = "proto3";
22
option cc_enable_arenas = true;
33

4+
import "src/api/protos/annotations/sensitive.proto";
45
import "src/api/protos/annotations/validation.proto";
56
import "src/api/protos/ydb_operation.proto";
67

@@ -86,23 +87,27 @@ message ExportToS3Settings {
8687
};
8788

8889
message Item {
89-
// Database path to a table to be exported
90+
// Database path to a table/directory to be exported
9091
string source_path = 1 [(required) = true];
9192

9293
/* Tables are exported to one or more S3 objects.
9394
The object name begins with 'destination_prefix'.
9495
This prefix will be followed by '/data_PartNumber', where 'PartNumber'
9596
represents the index of the part, starting at zero.
97+
Not required if the default `destination_prefix` is set.
98+
If not specified, actual S3 path is the default `destination_prefix` concatenated with:
99+
* The object path relative to the global `source_path` for a non-encrypted export
100+
* The anonymized path for an encrypted export
96101
*/
97-
string destination_prefix = 2 [(required) = true];
102+
string destination_prefix = 2;
98103
}
99104

100105
string endpoint = 1 [(required) = true];
101106
Scheme scheme = 2; // HTTPS if not specified
102107
string bucket = 3 [(required) = true];
103108
string access_key = 4 [(required) = true];
104109
string secret_key = 5 [(required) = true];
105-
repeated Item items = 6 [(size).ge = 1];
110+
repeated Item items = 6;
106111
string description = 7 [(length).le = 128];
107112
uint32 number_of_retries = 8;
108113
StorageClass storage_class = 9;
@@ -120,6 +125,20 @@ message ExportToS3Settings {
120125
// details: https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html
121126
// it is especially useful for custom s3 implementations
122127
bool disable_virtual_addressing = 12;
128+
129+
// Database root if not provided.
130+
// All object names are calculated and written relative to this path.
131+
string source_path = 13;
132+
133+
// A default S3 path prefix for all export items.
134+
// When specified, export writes SchemaMapping file with the list of objects.
135+
// Must be provided for an encrypted backup.
136+
string destination_prefix = 14;
137+
138+
// Settings for data encryption.
139+
// If encryption_settings field is not specified,
140+
// the resulting data will not be encrypted.
141+
EncryptionSettings encryption_settings = 15;
123142
}
124143

125144
message ExportToS3Result {
@@ -141,3 +160,24 @@ message ExportToS3Response {
141160
// operation.metadata = ExportToS3Metadata
142161
Ydb.Operations.Operation operation = 1;
143162
}
163+
164+
// Export encryption settings
165+
// Don't specify this struct for unencrypted exports
166+
message EncryptionSettings {
167+
// Algorithm for export encryption.
168+
// Not required in case of import/list operation.
169+
// Currently the following algorithms are supported:
170+
// AES-128-GCM
171+
// AES-256-GCM
172+
// ChaCha20-Poly1305
173+
string encryption_algorithm = 1;
174+
175+
oneof Key {
176+
SymmetricKey symmetric_key = 2;
177+
}
178+
179+
message SymmetricKey {
180+
// This key will be used for data encryption
181+
bytes key = 1 [(Ydb.sensitive) = true];
182+
}
183+
}

src/api/protos/ydb_import.proto

Lines changed: 35 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ syntax = "proto3";
22
option cc_enable_arenas = true;
33

44
import "src/api/protos/annotations/validation.proto";
5+
import "src/api/protos/ydb_export.proto";
56
import "src/api/protos/ydb_operation.proto";
67

78
import "google/protobuf/timestamp.proto";
@@ -39,25 +40,33 @@ message ImportFromS3Settings {
3940
}
4041

4142
message Item {
42-
/* YDB tables in S3 are stored in one or more objects (see ydb_export.proto).
43-
The object name begins with 'source_prefix'.
44-
This prefix is followed by:
45-
* '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
46-
* '/scheme.pb' - object with information about scheme, indexes, etc;
47-
* '/permissions.pb' - object with information about ACL and owner.
48-
*/
49-
string source_prefix = 1 [(required) = true];
50-
51-
// Database path to a table to import to.
52-
string destination_path = 2 [(required) = true];
43+
oneof Source {
44+
/* YDB database objects in S3 are stored in one or more S3 objects (see ydb_export.proto).
45+
The S3 object name begins with a prefix, followed by:
46+
* '/data_PartNumber', where 'PartNumber' represents the index of the part, starting at zero;
47+
* '/scheme.pb' - object with information about scheme, indexes, etc;
48+
* '/permissions.pb' - object with information about ACL and owner.
49+
*/
50+
51+
// The S3 object prefix can be either provided explicitly
52+
string source_prefix = 1;
53+
54+
// Or, if the export contains the database objects list, you may specify the database object name, and the S3 prefix will be looked up in the database objects list by the import procedure
55+
string source_path = 3;
56+
}
57+
58+
// Database path to a database object to import the item to
59+
// Resolved relative to the default destination_path
60+
// May be omitted if the item's source_path is specified, in this case will be taken equal to it
61+
string destination_path = 2;
5362
}
5463

5564
string endpoint = 1 [(required) = true];
5665
Scheme scheme = 2; // HTTPS if not specified
5766
string bucket = 3 [(required) = true];
5867
string access_key = 4 [(required) = true];
5968
string secret_key = 5 [(required) = true];
60-
repeated Item items = 6 [(size).ge = 1];
69+
repeated Item items = 6; // Empty collection means import of all export objects
6170
string description = 7 [(length).le = 128];
6271
uint32 number_of_retries = 8;
6372

@@ -76,6 +85,20 @@ message ImportFromS3Settings {
7685

7786
// Skip checksum validation during import
7887
bool skip_checksum_validation = 12;
88+
89+
// A default path prefix for all items,
90+
// determines that the import works with the list of objects in the SchemaMapping file.
91+
// Must be provided to import an encrypted export.
92+
string source_prefix = 13;
93+
94+
// Destination path to restore paths inside database
95+
// Default value is database root
96+
string destination_path = 14;
97+
98+
// Settings how data is encrypted.
99+
// If encryption_settings field is not specified,
100+
// the resulting data is considered not encrypted.
101+
Ydb.Export.EncryptionSettings encryption_settings = 15;
79102
}
80103

81104
message ImportFromS3Result {

src/api/protos/ydb_scheme.proto

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,7 @@ message Entry {
6666
EXTERNAL_DATA_SOURCE = 19;
6767
VIEW = 20;
6868
RESOURCE_POOL = 21;
69+
TRANSFER = 23;
6970
}
7071

7172
// Name of scheme entry (dir2 of /dir1/dir2)

src/client/table/table.cpp

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2040,6 +2040,14 @@ const std::optional<std::string>& TDataQuery::GetText() const {
20402040
return Impl_->GetText();
20412041
}
20422042

2043+
std::map<std::string, TType> TDataQuery::GetParameterTypes() const {
2044+
std::map<std::string, TType> typesMap;
2045+
for (const auto& param : Impl_->ParameterTypes_) {
2046+
typesMap.emplace(param.first, TType(param.second));
2047+
}
2048+
return typesMap;
2049+
}
2050+
20432051
TParamsBuilder TDataQuery::GetParamsBuilder() const {
20442052
return TParamsBuilder(Impl_->ParameterTypes_);
20452053
}

0 commit comments

Comments
 (0)