Skip to content

Commit d16ba9b

Browse files
GrigoriyPAdorooleg
GrigoriyPA
and
dorooleg
authored
merge to ydb stable aws credentials have been fixed (#14203)
Co-authored-by: Oleg Doronin <dorooleg@yandex.ru>
1 parent ef55dcd commit d16ba9b

File tree

4 files changed

+76
-8
lines changed

4 files changed

+76
-8
lines changed

‎ydb/core/external_sources/s3/ut/s3_aws_credentials_ut.cpp

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,30 @@ TString GetExternalPort(const TString& service, const TString& port) {
4444
return result ? Strip(result.back()) : TString{};
4545
}
4646

47+
void WaitBucket(std::shared_ptr<TKikimrRunner> kikimr, const TString& externalDataSourceName) {
48+
auto db = kikimr->GetQueryClient();
49+
for (size_t i = 0; i < 100; i++) {
50+
auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
51+
SELECT * FROM `{external_source}`.`/a/` WITH (
52+
format="json_each_row",
53+
schema(
54+
key Utf8 NOT NULL,
55+
value Utf8 NOT NULL
56+
)
57+
)
58+
)", "external_source"_a = externalDataSourceName)).ExtractValueSync();
59+
UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
60+
UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId);
61+
62+
NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
63+
if (readyOp.Metadata().ExecStatus == EExecStatus::Completed) {
64+
return;
65+
}
66+
Sleep(TDuration::Seconds(1));
67+
}
68+
UNIT_FAIL("Bucket isn't ready");
69+
}
70+
4771
Y_UNIT_TEST_SUITE(S3AwsCredentials) {
4872
Y_UNIT_TEST(ExecuteScriptWithEqSymbol) {
4973
const TString externalDataSourceName = "/Root/external_data_source";
@@ -69,6 +93,7 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) {
6993
);
7094
auto result = session.ExecuteSchemeQuery(query).GetValueSync();
7195
UNIT_ASSERT_C(result.GetStatus() == NYdb::EStatus::SUCCESS, result.GetIssues().ToString());
96+
WaitBucket(kikimr, externalDataSourceName);
7297
auto db = kikimr->GetQueryClient();
7398
{
7499
auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
@@ -183,6 +208,46 @@ Y_UNIT_TEST_SUITE(S3AwsCredentials) {
183208
UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2");
184209
UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world");
185210
}
211+
{
212+
auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
213+
SELECT * FROM `{external_source}`.`/` WITH (
214+
format="json_each_row",
215+
schema(
216+
key Utf8 NOT NULL,
217+
value Utf8 NOT NULL
218+
)
219+
)
220+
)", "external_source"_a = externalDataSourceName)).ExtractValueSync();
221+
UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
222+
UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId);
223+
224+
NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
225+
UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString());
226+
TFetchScriptResultsResult results = db.FetchScriptResults(scriptExecutionOperation.Id(), 0).ExtractValueSync();
227+
UNIT_ASSERT_C(results.IsSuccess(), results.GetIssues().ToString());
228+
229+
TResultSetParser resultSet(results.ExtractResultSet());
230+
UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnsCount(), 2);
231+
UNIT_ASSERT_VALUES_EQUAL(resultSet.RowsCount(), 4);
232+
UNIT_ASSERT(resultSet.TryNextRow());
233+
UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "1");
234+
UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "trololo");
235+
UNIT_ASSERT(resultSet.TryNextRow());
236+
UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(0).GetUtf8(), "2");
237+
UNIT_ASSERT_VALUES_EQUAL(resultSet.ColumnParser(1).GetUtf8(), "hello world");
238+
}
239+
240+
{
241+
auto scriptExecutionOperation = db.ExecuteScript(fmt::format(R"(
242+
INSERT INTO `{external_source}`.`exp_folder/` WITH (FORMAT = "csv_with_names")
243+
SELECT "Hello, world!" AS Data
244+
)", "external_source"_a = externalDataSourceName)).ExtractValueSync();
245+
UNIT_ASSERT_VALUES_EQUAL_C(scriptExecutionOperation.Status().GetStatus(), EStatus::SUCCESS, scriptExecutionOperation.Status().GetIssues().ToString());
246+
UNIT_ASSERT(scriptExecutionOperation.Metadata().ExecutionId);
247+
248+
NYdb::NQuery::TScriptExecutionOperation readyOp = WaitScriptExecutionOperation(scriptExecutionOperation.Id(), kikimr->GetDriver());
249+
UNIT_ASSERT_EQUAL_C(readyOp.Metadata().ExecStatus, EExecStatus::Completed, readyOp.Status().GetIssues().ToString());
250+
}
186251
}
187252

188253
}

‎ydb/core/external_sources/ya.make

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,4 +34,6 @@ RECURSE_FOR_TESTS(
3434

3535
RECURSE(
3636
hive_metastore
37+
object_storage
38+
s3
3739
)

‎ydb/library/yql/providers/common/http_gateway/yql_aws_signature.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ TString TAwsSignature::GetAuthorization() const {
6363
}
6464

6565
TString TAwsSignature::GetXAmzContentSha256() const {
66-
return HashSHA256(TStringBuilder{} << "\"" << Payload << "\"");
66+
return HashSHA256(TStringBuilder{} << Payload);
6767
}
6868

6969
TString TAwsSignature::GetAmzDate() const {
@@ -158,7 +158,8 @@ TString TAwsSignature::UriEncode(const TStringBuf input, bool encodeSlash, bool
158158
}
159159

160160
void TAwsSignature::PrepareCgiParameters() {
161-
TCgiParameters cgi(Cgi);
161+
TCgiParameters cgi;
162+
cgi.ScanAddAll(Cgi);
162163
TMap<TString, TVector<TString>> sortedCgi;
163164

164165
for (const auto& [key, value] : cgi) {

‎ydb/library/yql/providers/common/http_gateway/yql_aws_signature_ut.cpp

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ Y_UNIT_TEST_SUITE(TAwsSignature) {
1010
Y_UNIT_TEST(Sign) {
1111
NYql::TAwsSignature signature("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", {}, "key", "pwd");
1212
UNIT_ASSERT_VALUES_EQUAL(signature.GetContentType(), "application/json");
13-
UNIT_ASSERT_VALUES_EQUAL(signature.GetXAmzContentSha256(), "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126");
13+
UNIT_ASSERT_VALUES_EQUAL(signature.GetXAmzContentSha256(), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
1414
UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=");
1515
UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/");
1616
UNIT_ASSERT_STRING_CONTAINS(signature.GetAuthorization(), "///aws4_request");
@@ -41,7 +41,7 @@ Y_UNIT_TEST_SUITE(TAwsSignature) {
4141
NYql::TAwsSignature signature3("GET", "http://os.com/my-bucket/year=2024/day=03/", "application/json", "", "key2", "pwd", time);
4242
NYql::TAwsSignature signature4("POST", "http://os.com/my-bucket/year=2024/day=03/", "application/json", "", "key2", "pwd", time);
4343
static const TString CONTENT_TYPE = "application/json";
44-
static const TString X_AMZ_CONTENT_SHA_256 = "12ae32cb1ec02d01eda3581b127c1fee3b0dc53572ed6baf239721a03d82e126";
44+
static const TString X_AMZ_CONTENT_SHA_256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
4545
static const TString X_AMX_DATE = "19700101T000000Z";
4646
UNIT_ASSERT_VALUES_EQUAL(signature1.GetContentType(), CONTENT_TYPE);
4747
UNIT_ASSERT_VALUES_EQUAL(signature2.GetContentType(), CONTENT_TYPE);
@@ -55,10 +55,10 @@ Y_UNIT_TEST_SUITE(TAwsSignature) {
5555
UNIT_ASSERT_VALUES_EQUAL(signature2.GetAmzDate(), X_AMX_DATE);
5656
UNIT_ASSERT_VALUES_EQUAL(signature3.GetAmzDate(), X_AMX_DATE);
5757
UNIT_ASSERT_VALUES_EQUAL(signature4.GetAmzDate(), X_AMX_DATE);
58-
UNIT_ASSERT_VALUES_EQUAL(signature1.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=fba1374eb117fe9d00fc04bb1b709a3ea6f152232ac1f7dc49117a505f7e9f3f");
59-
UNIT_ASSERT_VALUES_EQUAL(signature2.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=39b47595c16b5d7b8256fd00e3d17e2157c5050c293306c995ae6980f11c689f");
60-
UNIT_ASSERT_VALUES_EQUAL(signature3.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=fba1374eb117fe9d00fc04bb1b709a3ea6f152232ac1f7dc49117a505f7e9f3f");
61-
UNIT_ASSERT_VALUES_EQUAL(signature4.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=a495d0ada1156d7278a56fb754a728d3b9470725208ad422bc299d6d6f793a8b");
58+
UNIT_ASSERT_VALUES_EQUAL(signature1.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=32e344ef8df96a217ac1780393acc92caf87e73238e444d290cc713b9e81f7e8");
59+
UNIT_ASSERT_VALUES_EQUAL(signature2.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=90e3a59639f80cff944aae40f3b4dba3435b3cf39b56e6395722469160c21f23");
60+
UNIT_ASSERT_VALUES_EQUAL(signature3.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=32e344ef8df96a217ac1780393acc92caf87e73238e444d290cc713b9e81f7e8");
61+
UNIT_ASSERT_VALUES_EQUAL(signature4.GetAuthorization(), "AWS4-HMAC-SHA256 Credential=/19700101///aws4_request, SignedHeaders=content-type;host;x-amz-content-sha256;x-amz-date, Signature=29af4a2f6e5be24030218e83c8a516a0511449951b078a9e4792a57e983d111a");
6262
}
6363

6464
Y_UNIT_TEST(SignWithCanonization) {

0 commit comments

Comments
 (0)