@@ -21,44 +21,131 @@ import (
2121 "fmt"
2222 "io"
2323 "io/ioutil"
24+ "math"
2425 "net/http"
26+ "strconv"
27+ "time"
28+
29+ "google.golang.org/api/option"
30+ "google.golang.org/api/transport"
2531)
2632
2733// HTTPClient is a convenient API to make HTTP calls.
2834//
29- // This API handles some of the repetitive tasks such as entity serialization and deserialization
30- // involved in making HTTP calls. It provides a convenient mechanism to set headers and query
35+ // This API handles repetitive tasks such as entity serialization and deserialization
36+ // when making HTTP calls. It provides a convenient mechanism to set headers and query
3137// parameters on outgoing requests, while enforcing that an explicit context is used per request.
32- // Responses returned by HTTPClient can be easily parsed as JSON, and provide a simple mechanism to
33- // extract error details.
38+ // Responses returned by HTTPClient can be easily unmarshalled as JSON.
39+ //
40+ // HTTPClient also handles automatically retrying failed HTTP requests.
3441type HTTPClient struct {
35- Client * http.Client
36- ErrParser ErrorParser
42+ Client * http.Client
43+ RetryConfig * RetryConfig
44+ ErrParser ErrorParser
3745}
3846
39- // Do executes the given Request, and returns a Response.
40- func (c * HTTPClient ) Do (ctx context.Context , r * Request ) (* Response , error ) {
41- req , err := r .buildHTTPRequest ()
47+ // NewHTTPClient creates a new HTTPClient using the provided client options and the default
48+ // RetryConfig.
49+ //
50+ // The default RetryConfig retries requests on all low-level network errors as well as on HTTP
51+ // InternalServerError (500) and ServiceUnavailable (503) errors. Repeatedly failing requests are
52+ // retried up to 4 times with exponential backoff. Retry delay is never longer than 2 minutes.
53+ //
54+ // NewHTTPClient returns the created HTTPClient along with the target endpoint URL. The endpoint
55+ // is obtained from the client options passed into the function.
56+ func NewHTTPClient (ctx context.Context , opts ... option.ClientOption ) (* HTTPClient , string , error ) {
57+ hc , endpoint , err := transport .NewHTTPClient (ctx , opts ... )
4258 if err != nil {
43- return nil , err
59+ return nil , "" , err
4460 }
61+ twoMinutes := time .Duration (2 ) * time .Minute
62+ client := & HTTPClient {
63+ Client : hc ,
64+ RetryConfig : & RetryConfig {
65+ MaxRetries : 4 ,
66+ CheckForRetry : retryNetworkAndHTTPErrors (
67+ http .StatusInternalServerError ,
68+ http .StatusServiceUnavailable ,
69+ ),
70+ ExpBackoffFactor : 0.5 ,
71+ MaxDelay : & twoMinutes ,
72+ },
73+ }
74+ return client , endpoint , nil
75+ }
4576
46- resp , err := c .Client .Do (req .WithContext (ctx ))
47- if err != nil {
48- return nil , err
77+ // Do executes the given Request, and returns a Response.
78+ //
79+ // If a RetryConfig is specified on the client, Do attempts to retry failing requests.
80+ func (c * HTTPClient ) Do (ctx context.Context , req * Request ) (* Response , error ) {
81+ var result * attemptResult
82+ var err error
83+
84+ for retries := 0 ; ; retries ++ {
85+ result , err = c .attempt (ctx , req , retries )
86+ if err != nil {
87+ return nil , err
88+ }
89+ if ! result .Retry {
90+ break
91+ }
92+ if err = result .waitForRetry (ctx ); err != nil {
93+ return nil , err
94+ }
4995 }
50- defer resp .Body .Close ()
96+ return result .handleResponse ()
97+ }
5198
52- b , err := ioutil .ReadAll (resp .Body )
99+ func (c * HTTPClient ) attempt (ctx context.Context , req * Request , retries int ) (* attemptResult , error ) {
100+ hr , err := req .buildHTTPRequest ()
53101 if err != nil {
54102 return nil , err
55103 }
56- return & Response {
57- Status : resp .StatusCode ,
58- Body : b ,
59- Header : resp .Header ,
60- errParser : c .ErrParser ,
61- }, nil
104+
105+ resp , err := c .Client .Do (hr .WithContext (ctx ))
106+ result := & attemptResult {
107+ Resp : resp ,
108+ Err : err ,
109+ ErrParser : c .ErrParser ,
110+ }
111+
112+ // If a RetryConfig is available, always consult it to determine if the request should be retried
113+ // or not. Even if there was a network error, we may not want to retry the request based on the
114+ // RetryConfig that is in effect.
115+ if c .RetryConfig != nil {
116+ delay , retry := c .RetryConfig .retryDelay (retries , resp , err )
117+ result .RetryAfter = delay
118+ result .Retry = retry
119+ if retry && resp != nil {
120+ defer resp .Body .Close ()
121+ }
122+ }
123+ return result , nil
124+ }
125+
126+ type attemptResult struct {
127+ Resp * http.Response
128+ Err error
129+ Retry bool
130+ RetryAfter time.Duration
131+ ErrParser ErrorParser
132+ }
133+
134+ func (r * attemptResult ) waitForRetry (ctx context.Context ) error {
135+ if r .RetryAfter > 0 {
136+ select {
137+ case <- ctx .Done ():
138+ case <- time .After (r .RetryAfter ):
139+ }
140+ }
141+ return ctx .Err ()
142+ }
143+
144+ func (r * attemptResult ) handleResponse () (* Response , error ) {
145+ if r .Err != nil {
146+ return nil , r .Err
147+ }
148+ return newResponse (r .Resp , r .ErrParser )
62149}
63150
64151// Request contains all the parameters required to construct an outgoing HTTP request.
@@ -124,9 +211,23 @@ type Response struct {
124211 errParser ErrorParser
125212}
126213
214+ func newResponse (resp * http.Response , errParser ErrorParser ) (* Response , error ) {
215+ defer resp .Body .Close ()
216+ b , err := ioutil .ReadAll (resp .Body )
217+ if err != nil {
218+ return nil , err
219+ }
220+ return & Response {
221+ Status : resp .StatusCode ,
222+ Body : b ,
223+ Header : resp .Header ,
224+ errParser : errParser ,
225+ }, nil
226+ }
227+
127228// CheckStatus checks whether the Response status code has the given HTTP status code.
128229//
129- // Returns an error if the status code does not match. If an ErroParser is specified, uses that to
230+ // Returns an error if the status code does not match. If an ErrorParser is specified, uses that to
130231// construct the returned error message. Otherwise includes the full response body in the error.
131232func (r * Response ) CheckStatus (want int ) error {
132233 if r .Status == want {
@@ -188,3 +289,93 @@ func WithQueryParams(qp map[string]string) HTTPOption {
188289 r .URL .RawQuery = q .Encode ()
189290 }
190291}
292+
293+ // RetryConfig specifies how the HTTPClient should retry failing HTTP requests.
294+ //
295+ // A request is never retried more than MaxRetries times. If CheckForRetry is nil, all network
296+ // errors, and all 400+ HTTP status codes are retried. If an HTTP error response contains the
297+ // Retry-After header, it is always respected. Otherwise retries are delayed with exponential
298+ // backoff. Set ExpBackoffFactor to 0 to disable exponential backoff, and retry immediately
299+ // after each error.
300+ //
301+ // If MaxDelay is set, retries delay gets capped by that value. If the Retry-After header
302+ // requires a longer delay than MaxDelay, retries are not attempted.
303+ type RetryConfig struct {
304+ MaxRetries int
305+ CheckForRetry RetryCondition
306+ ExpBackoffFactor float64
307+ MaxDelay * time.Duration
308+ }
309+
310+ // RetryCondition determines if an HTTP request should be retried depending on its last outcome.
311+ type RetryCondition func (resp * http.Response , networkErr error ) bool
312+
313+ func (rc * RetryConfig ) retryDelay (retries int , resp * http.Response , err error ) (time.Duration , bool ) {
314+ if ! rc .retryEligible (retries , resp , err ) {
315+ return 0 , false
316+ }
317+ estimatedDelay := rc .estimateDelayBeforeNextRetry (retries )
318+ serverRecommendedDelay := parseRetryAfterHeader (resp )
319+ if serverRecommendedDelay > estimatedDelay {
320+ estimatedDelay = serverRecommendedDelay
321+ }
322+ if rc .MaxDelay != nil && estimatedDelay > * rc .MaxDelay {
323+ return 0 , false
324+ }
325+ return estimatedDelay , true
326+ }
327+
328+ func (rc * RetryConfig ) retryEligible (retries int , resp * http.Response , err error ) bool {
329+ if retries >= rc .MaxRetries {
330+ return false
331+ }
332+ if rc .CheckForRetry == nil {
333+ return err != nil || resp .StatusCode >= 500
334+ }
335+ return rc .CheckForRetry (resp , err )
336+ }
337+
338+ func (rc * RetryConfig ) estimateDelayBeforeNextRetry (retries int ) time.Duration {
339+ if retries == 0 {
340+ return 0
341+ }
342+ delayInSeconds := int64 (math .Pow (2 , float64 (retries )) * rc .ExpBackoffFactor )
343+ estimatedDelay := time .Duration (delayInSeconds ) * time .Second
344+ if rc .MaxDelay != nil && estimatedDelay > * rc .MaxDelay {
345+ estimatedDelay = * rc .MaxDelay
346+ }
347+ return estimatedDelay
348+ }
349+
350+ var retryTimeClock Clock = & SystemClock {}
351+
352+ func parseRetryAfterHeader (resp * http.Response ) time.Duration {
353+ if resp == nil {
354+ return 0
355+ }
356+ retryAfterHeader := resp .Header .Get ("retry-after" )
357+ if retryAfterHeader == "" {
358+ return 0
359+ }
360+ if delayInSeconds , err := strconv .ParseInt (retryAfterHeader , 10 , 64 ); err == nil {
361+ return time .Duration (delayInSeconds ) * time .Second
362+ }
363+ if timestamp , err := http .ParseTime (retryAfterHeader ); err == nil {
364+ return timestamp .Sub (retryTimeClock .Now ())
365+ }
366+ return 0
367+ }
368+
369+ func retryNetworkAndHTTPErrors (statusCodes ... int ) RetryCondition {
370+ return func (resp * http.Response , networkErr error ) bool {
371+ if networkErr != nil {
372+ return true
373+ }
374+ for _ , retryOnStatus := range statusCodes {
375+ if resp .StatusCode == retryOnStatus {
376+ return true
377+ }
378+ }
379+ return false
380+ }
381+ }
0 commit comments