Skip to content

Commit e4a6ceb

Browse files
committed
YQ-3439 added retries for CURLE_COULDNT_RESOLVE_HOST (ydb-platform#6732)
1 parent fe5e16f commit e4a6ceb

15 files changed

+55
-21
lines changed

ydb/core/external_sources/object_storage.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,8 @@ struct TObjectStorageExternalSource : public IExternalSource {
304304
}
305305

306306
auto httpGateway = NYql::IHTTPGateway::Make();
307-
auto s3Lister = NYql::NS3Lister::MakeS3Lister(httpGateway, NYql::NS3Lister::TListingRequest{
307+
auto httpRetryPolicy = NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.RetriedCurlCodes = NYql::FqRetriedCurlCodes()});
308+
auto s3Lister = NYql::NS3Lister::MakeS3Lister(httpGateway, httpRetryPolicy, NYql::NS3Lister::TListingRequest{
308309
.Url = meta->DataSourceLocation,
309310
.AuthInfo = authInfo,
310311
.Pattern = meta->TableLocation,

ydb/core/kqp/host/kqp_host.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1518,6 +1518,7 @@ class TKqpHost : public IKqpHost {
15181518
state->Configuration->AllowAtomicUploadCommit = queryType == EKikimrQueryType::Script;
15191519
state->Configuration->Init(FederatedQuerySetup->S3GatewayConfig, TypesCtx);
15201520
state->Gateway = FederatedQuerySetup->HttpGateway;
1521+
state->GatewayRetryPolicy = NYql::GetHTTPDefaultRetryPolicy(NYql::THttpRetryPolicyOptions{.RetriedCurlCodes = NYql::FqRetriedCurlCodes()});
15211522
state->ExecutorPoolId = AppData()->UserPoolId;
15221523

15231524
auto dataSource = NYql::CreateS3DataSource(state);

ydb/library/yql/providers/common/http_gateway/yql_http_default_retry_policy.cpp

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,8 @@ std::unordered_set<CURLcode> FqRetriedCurlCodes() {
2929
CURLE_SEND_ERROR,
3030
CURLE_RECV_ERROR,
3131
CURLE_NO_CONNECTION_AVAILABLE,
32-
CURLE_GOT_NOTHING
32+
CURLE_GOT_NOTHING,
33+
CURLE_COULDNT_RESOLVE_HOST
3334
};
3435
}
3536

ydb/library/yql/providers/s3/actors/yql_s3_raw_read_actor.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ class TS3ReadActor : public NActors::TActorBootstrapped<TS3ReadActor>, public ID
111111
FileQueueBatchSizeLimit,
112112
FileQueueBatchObjectCountLimit,
113113
Gateway,
114+
RetryPolicy,
114115
Url,
115116
AuthInfo,
116117
Pattern,

ydb/library/yql/providers/s3/actors/yql_s3_read_actor.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1315,6 +1315,7 @@ class TS3StreamReadActor : public TActorBootstrapped<TS3StreamReadActor>, public
13151315
FileQueueBatchSizeLimit,
13161316
FileQueueBatchObjectCountLimit,
13171317
Gateway,
1318+
RetryPolicy,
13181319
Url,
13191320
AuthInfo,
13201321
Pattern,

ydb/library/yql/providers/s3/actors/yql_s3_read_actor.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ NActors::IActor* CreateS3FileQueueActor(
2525
ui64 batchSizeLimit,
2626
ui64 batchObjectCountLimit,
2727
IHTTPGateway::TPtr gateway,
28+
IHTTPGateway::TRetryPolicy::TPtr retryPolicy,
2829
TString url,
2930
TS3Credentials::TAuthInfo authInfo,
3031
TString pattern,

ydb/library/yql/providers/s3/actors/yql_s3_source_queue.cpp

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ class TS3FileQueueActor : public NActors::TActorBootstrapped<TS3FileQueueActor>
171171
ui64 batchSizeLimit,
172172
ui64 batchObjectCountLimit,
173173
IHTTPGateway::TPtr gateway,
174+
IHTTPGateway::TRetryPolicy::TPtr retryPolicy,
174175
TString url,
175176
TS3Credentials::TAuthInfo authInfo,
176177
TString pattern,
@@ -186,6 +187,7 @@ class TS3FileQueueActor : public NActors::TActorBootstrapped<TS3FileQueueActor>
186187
, BatchSizeLimit(batchSizeLimit)
187188
, BatchObjectCountLimit(batchObjectCountLimit)
188189
, Gateway(std::move(gateway))
190+
, RetryPolicy(std::move(retryPolicy))
189191
, Url(std::move(url))
190192
, AuthInfo(std::move(authInfo))
191193
, Pattern(std::move(pattern))
@@ -488,6 +490,7 @@ class TS3FileQueueActor : public NActors::TActorBootstrapped<TS3FileQueueActor>
488490
CurrentDirectoryPathIndex = object.GetPathIndex();
489491
MaybeLister = NS3Lister::MakeS3Lister(
490492
Gateway,
493+
RetryPolicy,
491494
NS3Lister::TListingRequest{
492495
Url,
493496
AuthInfo,
@@ -611,6 +614,7 @@ class TS3FileQueueActor : public NActors::TActorBootstrapped<TS3FileQueueActor>
611614
THashSet<NActors::TActorId> UpdatedConsumers;
612615

613616
const IHTTPGateway::TPtr Gateway;
617+
const IHTTPGateway::TRetryPolicy::TPtr RetryPolicy;
614618
const TString Url;
615619
const TS3Credentials::TAuthInfo AuthInfo;
616620
const TString Pattern;
@@ -632,6 +636,7 @@ NActors::IActor* CreateS3FileQueueActor(
632636
ui64 batchSizeLimit,
633637
ui64 batchObjectCountLimit,
634638
IHTTPGateway::TPtr gateway,
639+
IHTTPGateway::TRetryPolicy::TPtr retryPolicy,
635640
TString url,
636641
TS3Credentials::TAuthInfo authInfo,
637642
TString pattern,
@@ -648,6 +653,7 @@ NActors::IActor* CreateS3FileQueueActor(
648653
batchSizeLimit,
649654
batchObjectCountLimit,
650655
gateway,
656+
retryPolicy,
651657
url,
652658
authInfo,
653659
pattern,

ydb/library/yql/providers/s3/actors/yql_s3_source_queue.h

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ NActors::IActor* CreateS3FileQueueActor(
2222
ui64 batchSizeLimit,
2323
ui64 batchObjectCountLimit,
2424
IHTTPGateway::TPtr gateway,
25+
IHTTPGateway::TRetryPolicy::TPtr retryPolicy,
2526
TString url,
2627
TS3Credentials::TAuthInfo authInfo,
2728
TString pattern,

ydb/library/yql/providers/s3/object_listers/yql_s3_list.cpp

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -243,6 +243,7 @@ class TS3Lister : public IS3Lister {
243243

244244
TS3Lister(
245245
const IHTTPGateway::TPtr& httpGateway,
246+
const IHTTPGateway::TRetryPolicy::TPtr& retryPolicy,
246247
const TListingRequest& listingRequest,
247248
const TMaybe<TString>& delimiter,
248249
size_t maxFilesPerQuery,
@@ -265,7 +266,7 @@ class TS3Lister : public IS3Lister {
265266
NewPromise<TMaybe<TListingContext>>(),
266267
std::make_shared<TListEntries>(),
267268
IHTTPGateway::TWeakPtr(httpGateway),
268-
GetHTTPDefaultRetryPolicy(),
269+
retryPolicy,
269270
CreateGuidAsString(),
270271
std::move(request),
271272
delimiter,
@@ -391,7 +392,7 @@ class TS3Lister : public IS3Lister {
391392
NewPromise<TMaybe<TListingContext>>(),
392393
std::make_shared<TListEntries>(),
393394
ctx.GatewayWeak,
394-
GetHTTPDefaultRetryPolicy(),
395+
ctx.RetryPolicy,
395396
CreateGuidAsString(),
396397
ctx.ListingRequest,
397398
ctx.Delimiter,
@@ -457,15 +458,16 @@ class TS3ParallelLimitedListerFactory : public IS3ListerFactory {
457458

458459
TFuture<NS3Lister::IS3Lister::TPtr> Make(
459460
const IHTTPGateway::TPtr& httpGateway,
461+
const IHTTPGateway::TRetryPolicy::TPtr& retryPolicy,
460462
const NS3Lister::TListingRequest& listingRequest,
461463
const TMaybe<TString>& delimiter,
462464
bool allowLocalFiles) override {
463465
auto acquired = Semaphore->AcquireAsync();
464466
return acquired.Apply(
465-
[ctx = SharedCtx, httpGateway, listingRequest, delimiter, allowLocalFiles](const auto& f) {
467+
[ctx = SharedCtx, httpGateway, retryPolicy, listingRequest, delimiter, allowLocalFiles](const auto& f) {
466468
return std::shared_ptr<NS3Lister::IS3Lister>(new TListerLockReleaseWrapper{
467469
NS3Lister::MakeS3Lister(
468-
httpGateway, listingRequest, delimiter, allowLocalFiles, ctx),
470+
httpGateway, retryPolicy, listingRequest, delimiter, allowLocalFiles, ctx),
469471
std::make_unique<TAsyncSemaphore::TAutoRelease>(
470472
f.GetValue()->MakeAutoRelease())});
471473
});
@@ -507,13 +509,14 @@ class TS3ParallelLimitedListerFactory : public IS3ListerFactory {
507509

508510
IS3Lister::TPtr MakeS3Lister(
509511
const IHTTPGateway::TPtr& httpGateway,
512+
const IHTTPGateway::TRetryPolicy::TPtr& retryPolicy,
510513
const TListingRequest& listingRequest,
511514
const TMaybe<TString>& delimiter,
512515
bool allowLocalFiles,
513516
TSharedListingContextPtr sharedCtx) {
514517
if (listingRequest.Url.substr(0, 7) != "file://") {
515518
return std::make_shared<TS3Lister>(
516-
httpGateway, listingRequest, delimiter, 1000, std::move(sharedCtx));
519+
httpGateway, retryPolicy, listingRequest, delimiter, 1000, std::move(sharedCtx));
517520
}
518521

519522
if (!allowLocalFiles) {

ydb/library/yql/providers/s3/object_listers/yql_s3_list.h

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@ class IS3Lister : public TIterator<NThreading::TFuture<TListResult>> {
165165

166166
IS3Lister::TPtr MakeS3Lister(
167167
const IHTTPGateway::TPtr& httpGateway,
168+
const IHTTPGateway::TRetryPolicy::TPtr& retryPolicy,
168169
const TListingRequest& listingRequest,
169170
const TMaybe<TString>& delimiter,
170171
bool allowLocalFiles,
@@ -176,6 +177,7 @@ class IS3ListerFactory {
176177

177178
virtual NThreading::TFuture<NS3Lister::IS3Lister::TPtr> Make(
178179
const IHTTPGateway::TPtr& httpGateway,
180+
const IHTTPGateway::TRetryPolicy::TPtr& retryPolicy,
179181
const NS3Lister::TListingRequest& listingRequest,
180182
const TMaybe<TString>& delimiter,
181183
bool allowLocalFiles) = 0;

0 commit comments

Comments
 (0)