Skip to content

Commit 99247a8

Browse files
jackofallopsjackofallops
committed
add dropped connection detection and retry handling for LRO and provisioning state
1 parent 92ddd11 commit 99247a8

File tree

5 files changed

+217
-188
lines changed

5 files changed

+217
-188
lines changed

‎sdk/client/client.go

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -779,7 +779,7 @@ func extendedRetryPolicy(resp *http.Response, err error) (bool, error) {
779779
// connection
780780
tcpDialTCPRe := regexp.MustCompile(`dial tcp`)
781781

782-
// A regular expression to match complete packet loss - see comment below on packet-loss scenarios
782+
// A regular expression to match complete packet loss
783783
completePacketLossRe := regexp.MustCompile(`EOF`)
784784

785785
if err != nil {
@@ -809,7 +809,6 @@ func extendedRetryPolicy(resp *http.Response, err error) (bool, error) {
809809
return false, v
810810
}
811811

812-
// Such as Temporary Proxy outage, or recoverable disruption to network traffic (e.g. bgp events, Proxy failures etc)
813812
if completePacketLossRe.MatchString(v.Error()) {
814813
return false, v
815814
}

‎sdk/client/resourcemanager/poller_lro.go

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"bytes"
88
"context"
99
"encoding/json"
10+
"errors"
1011
"fmt"
1112
"io"
1213
"net/http"
@@ -27,6 +28,9 @@ type longRunningOperationPoller struct {
2728
initialRetryDuration time.Duration
2829
originalUrl *url.URL
2930
pollingUrl *url.URL
31+
32+
droppedConnectionCount int
33+
maxDroppedConnections int
3034
}
3135

3236
func pollingUriForLongRunningOperation(resp *client.Response) string {
@@ -39,8 +43,9 @@ func pollingUriForLongRunningOperation(resp *client.Response) string {
3943

4044
func longRunningOperationPollerFromResponse(resp *client.Response, client *client.Client) (*longRunningOperationPoller, error) {
4145
poller := longRunningOperationPoller{
42-
client: client,
43-
initialRetryDuration: 10 * time.Second,
46+
client: client,
47+
initialRetryDuration: 10 * time.Second,
48+
maxDroppedConnections: 3,
4449
}
4550

4651
pollingUrl := pollingUriForLongRunningOperation(resp)
@@ -107,9 +112,20 @@ func (p *longRunningOperationPoller) Poll(ctx context.Context) (result *pollers.
107112
}
108113
result.HttpResponse, err = req.Execute(ctx)
109114
if err != nil {
115+
var e *url.Error
116+
if errors.As(err, &e) {
117+
p.droppedConnectionCount++
118+
if p.droppedConnectionCount < p.maxDroppedConnections {
119+
result.Status = pollers.PollingStatusUnknown
120+
return result, nil
121+
}
122+
}
123+
110124
return nil, err
111125
}
112126

127+
p.droppedConnectionCount = 0
128+
113129
if result.HttpResponse != nil {
114130
var respBody []byte
115131
respBody, err = io.ReadAll(result.HttpResponse.Body)

‎sdk/client/resourcemanager/poller_lro_test.go

Lines changed: 80 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -304,85 +304,86 @@ func TestPollerLRO_InStatus_AcceptedThenInProgressThenSuccess(t *testing.T) {
304304
helpers.assertCalled(t, 3)
305305
}
306306

307-
//func TestPollerLRO_InProvisioningState_AcceptedThenDroppedThenInProgressThenSuccess(t *testing.T) {
308-
// ctx := context.TODO()
309-
// helpers := newLongRunningOperationsEndpoint([]expectedResponse{
310-
// responseWithHttpStatusCode(http.StatusAccepted),
311-
// responseThatDropsTheConnection(),
312-
// responseWithStatusInProvisioningState(statusInProgress),
313-
// responseWithStatusInProvisioningState(statusSucceeded),
314-
// })
315-
// server := httptest.NewServer(http.HandlerFunc(helpers.endpoint(t)))
316-
// defer server.Close()
317-
//
318-
// response := &client.Response{
319-
// Response: helpers.response(),
320-
// }
321-
// client := client.NewClient(server.URL, "MyService", "2020-02-01")
322-
// poller, err := longRunningOperationPollerFromResponse(response, client)
323-
// if err != nil {
324-
// t.Fatal(err.Error())
325-
// }
326-
//
327-
// expectedStatuses := []pollers.PollingStatus{
328-
// pollers.PollingStatusInProgress, // the 202 Accepted
329-
// // NOTE: the Dropped Connection will be ignored/silently retried
330-
// pollers.PollingStatusInProgress, // working on it
331-
// pollers.PollingStatusSucceeded, // good
332-
// }
333-
// for i, expected := range expectedStatuses {
334-
// t.Logf("Poll %d..", i)
335-
// result, err := poller.Poll(ctx)
336-
// if err != nil {
337-
// t.Fatal(err.Error())
338-
// }
339-
// if result.Status != expected {
340-
// t.Fatalf("expected status to be %q but got %q", expected, result.Status)
341-
// }
342-
// }
343-
// // sanity-checking - expect 4 calls but 3 statuses (since the dropped connection is silently retried)
344-
// helpers.assertCalled(t, 4)
345-
//}
346-
347-
//func TestPollerLRO_InStatus_AcceptedThenDroppedThenInProgressThenSuccess(t *testing.T) {
348-
// ctx := context.TODO()
349-
// helpers := newLongRunningOperationsEndpoint([]expectedResponse{
350-
// responseWithHttpStatusCode(http.StatusAccepted),
351-
// responseThatDropsTheConnection(),
352-
// responseWithStatusInStatusField(statusInProgress),
353-
// responseWithStatusInStatusField(statusSucceeded),
354-
// })
355-
// server := httptest.NewServer(http.HandlerFunc(helpers.endpoint(t)))
356-
// defer server.Close()
357-
//
358-
// response := &client.Response{
359-
// Response: helpers.response(),
360-
// }
361-
// client := client.NewClient(server.URL, "MyService", "2020-02-01")
362-
// poller, err := longRunningOperationPollerFromResponse(response, client)
363-
// if err != nil {
364-
// t.Fatal(err.Error())
365-
// }
366-
//
367-
// expectedStatuses := []pollers.PollingStatus{
368-
// pollers.PollingStatusInProgress, // the 202 Accepted
369-
// // NOTE: the Dropped Connection will be ignored/silently retried
370-
// pollers.PollingStatusInProgress, // working on it
371-
// pollers.PollingStatusSucceeded, // good
372-
// }
373-
// for i, expected := range expectedStatuses {
374-
// t.Logf("Poll %d..", i)
375-
// result, err := poller.Poll(ctx)
376-
// if err != nil {
377-
// t.Fatal(err.Error())
378-
// }
379-
// if result.Status != expected {
380-
// t.Fatalf("expected status to be %q but got %q", expected, result.Status)
381-
// }
382-
// }
383-
// // sanity-checking - expect 4 calls but 3 statuses (since the dropped connection is silently retried)
384-
// helpers.assertCalled(t, 4)
385-
//}
307+
func TestPollerLRO_InProvisioningState_AcceptedThenDroppedThenInProgressThenSuccess(t *testing.T) {
308+
ctx := context.TODO()
309+
helpers := newLongRunningOperationsEndpoint([]expectedResponse{
310+
responseWithHttpStatusCode(http.StatusAccepted),
311+
responseThatDropsTheConnection(),
312+
responseWithStatusInProvisioningState(statusInProgress),
313+
responseWithStatusInProvisioningState(statusSucceeded),
314+
})
315+
server := httptest.NewServer(http.HandlerFunc(helpers.endpoint(t)))
316+
defer server.Close()
317+
318+
response := &client.Response{
319+
Response: helpers.response(),
320+
}
321+
client := client.NewClient(server.URL, "MyService", "2020-02-01")
322+
poller, err := longRunningOperationPollerFromResponse(response, client)
323+
if err != nil {
324+
t.Fatal(err.Error())
325+
}
326+
327+
expectedStatuses := []pollers.PollingStatus{
328+
pollers.PollingStatusInProgress, // the 202 Accepted
329+
pollers.PollingStatusUnknown,
330+
// NOTE: the Dropped Connection will be ignored/silently retried
331+
pollers.PollingStatusInProgress, // working on it
332+
pollers.PollingStatusSucceeded, // good
333+
}
334+
for i, expected := range expectedStatuses {
335+
t.Logf("Poll %d..", i)
336+
result, err := poller.Poll(ctx)
337+
if err != nil {
338+
t.Fatal(err.Error())
339+
}
340+
if result.Status != expected {
341+
t.Fatalf("expected status to be %q but got %q", expected, result.Status)
342+
}
343+
}
344+
// sanity-checking - expect 4 calls but 3 statuses (since the dropped connection is silently retried)
345+
helpers.assertCalled(t, 4)
346+
}
347+
348+
func TestPollerLRO_InStatus_AcceptedThenDroppedThenInProgressThenSuccess(t *testing.T) {
349+
ctx := context.TODO()
350+
helpers := newLongRunningOperationsEndpoint([]expectedResponse{
351+
responseWithHttpStatusCode(http.StatusAccepted),
352+
responseThatDropsTheConnection(),
353+
responseWithStatusInStatusField(statusInProgress),
354+
responseWithStatusInStatusField(statusSucceeded),
355+
})
356+
server := httptest.NewServer(http.HandlerFunc(helpers.endpoint(t)))
357+
defer server.Close()
358+
359+
response := &client.Response{
360+
Response: helpers.response(),
361+
}
362+
client := client.NewClient(server.URL, "MyService", "2020-02-01")
363+
poller, err := longRunningOperationPollerFromResponse(response, client)
364+
if err != nil {
365+
t.Fatal(err.Error())
366+
}
367+
368+
expectedStatuses := []pollers.PollingStatus{
369+
pollers.PollingStatusInProgress, // the 202 Accepted
370+
pollers.PollingStatusUnknown,
371+
pollers.PollingStatusInProgress, // working on it
372+
pollers.PollingStatusSucceeded, // good
373+
}
374+
for i, expected := range expectedStatuses {
375+
t.Logf("Poll %d..", i)
376+
result, err := poller.Poll(ctx)
377+
if err != nil {
378+
t.Fatal(err.Error())
379+
}
380+
if result.Status != expected {
381+
t.Fatalf("expected status to be %q but got %q", expected, result.Status)
382+
}
383+
}
384+
// sanity-checking - expect 4 calls but 3 statuses (since the dropped connection is silently retried)
385+
helpers.assertCalled(t, 4)
386+
}
386387

387388
func TestPollerLRO_InProvisioningState_404ThenImmediateSuccess(t *testing.T) {
388389
// This scenario handles the API returning a 404 initially, then succeeded

‎sdk/client/resourcemanager/poller_provisioning_state.go

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ package resourcemanager
55

66
import (
77
"context"
8+
"errors"
89
"fmt"
910
"net/http"
1011
"net/url"
@@ -27,6 +28,9 @@ type provisioningStatePoller struct {
2728
initialRetryDuration time.Duration
2829
originalUri string
2930
resourcePath string
31+
32+
droppedConnectionCount int
33+
maxDroppedConnections int
3034
}
3135

3236
func provisioningStatePollerFromResponse(response *client.Response, lroIsSelfReference bool, client *Client, pollingInterval time.Duration) (*provisioningStatePoller, error) {
@@ -58,11 +62,12 @@ func provisioningStatePollerFromResponse(response *client.Response, lroIsSelfRef
5862
}
5963

6064
return &provisioningStatePoller{
61-
apiVersion: apiVersion,
62-
client: client,
63-
initialRetryDuration: pollingInterval,
64-
originalUri: originalUri,
65-
resourcePath: resourcePath,
65+
apiVersion: apiVersion,
66+
client: client,
67+
initialRetryDuration: pollingInterval,
68+
originalUri: originalUri,
69+
resourcePath: resourcePath,
70+
maxDroppedConnections: 3,
6671
}, nil
6772
}
6873

@@ -84,6 +89,17 @@ func (p *provisioningStatePoller) Poll(ctx context.Context) (*pollers.PollResult
8489
}
8590
resp, err := p.client.Execute(ctx, req)
8691
if err != nil {
92+
var e *url.Error
93+
if errors.As(err, &e) {
94+
p.droppedConnectionCount++
95+
if p.droppedConnectionCount < p.maxDroppedConnections {
96+
return &pollers.PollResult{
97+
PollInterval: p.initialRetryDuration,
98+
Status: pollers.PollingStatusUnknown,
99+
}, nil
100+
}
101+
}
102+
87103
return nil, fmt.Errorf("executing request: %+v", err)
88104
}
89105
if resp == nil {

0 commit comments

Comments
 (0)