fix retry middleware body handling for POST/PUT/PATCH requests

- add proper request body buffering for retries when GetBody is not set
- buffering disabled by default to preserve streaming and backward compatibility
- configurable via RetryBufferBodies(true) and RetryMaxBufferSize options
- prevent sending empty bodies when original request had content
- limit retries to single attempt when body cannot be replayed
- add comprehensive tests for body handling scenarios
- update documentation to clarify body replay behavior

Author: umputun

README.md

@@ -79,6 +79,8 @@ rq := requester.New(http.Client{}, middleware.Retry(3, 100*time.Millisecond,
     middleware.RetryWithJitter(0.1),                          // add 10% randomization
     middleware.RetryOnCodes(503, 502),                        // retry only on specific codes
     // or middleware.RetryExcludeCodes(404, 401),             // alternatively, retry on all except these codes
+    middleware.RetryBufferBodies(true),                       // enable body buffering for retries
+    middleware.RetryMaxBufferSize(50*1024*1024),             // set max buffer size to 50MB
 ))
 ```
 
@@ -89,13 +91,29 @@ Default configuration:
 - Exponential backoff
 - 10% jitter
 - Retries on 5xx status codes
+- Body buffering: disabled (preserves streaming behavior)
+
+#### Request Body Handling
+
+For requests with bodies (POST, PUT, PATCH), the retry middleware handles body replay as follows:
+
+- If `req.GetBody` is set (automatic for `strings.Reader`, `bytes.Buffer`, `bytes.Reader`), it uses that for retries
+- If `req.GetBody` is nil and body buffering is disabled (default), requests won't be retried to preserve streaming
+- If body buffering is enabled with `RetryBufferBodies(true)`:
+  - Bodies up to `maxBufferSize` are buffered and can be retried
+  - Bodies exceeding `maxBufferSize` fail immediately with an error
+- Use `RetryMaxBufferSize(size)` to adjust the buffer limit (default: 10MB when enabled)
+
+This default behavior ensures large file uploads work without memory issues, while you can opt-in to buffering for API requests that need retries.
 
 Retry Options:
 - `RetryWithBackoff(t BackoffType)` - set backoff strategy (Constant, Linear, or Exponential)
 - `RetryMaxDelay(d time.Duration)` - cap the maximum delay between retries
 - `RetryWithJitter(f float64)` - add randomization to delays (0-1.0 factor)
 - `RetryOnCodes(codes ...int)` - retry only on specific status codes
 - `RetryExcludeCodes(codes ...int)` - retry on all codes except specified
+- `RetryBufferBodies(enabled bool)` - enable or disable automatic body buffering for retries
+- `RetryMaxBufferSize(size int64)` - set maximum size of request bodies that will be buffered
 
 Note: `RetryOnCodes` and `RetryExcludeCodes` are mutually exclusive and can't be used together.
 

middleware/retry.go

@@ -1,7 +1,9 @@
 package middleware
 
 import (
+	"bytes"
 	"fmt"
+	"io"
 	"math"
 	"math/rand"
 	"net/http"
@@ -25,34 +27,44 @@ const (
 // By default retries on network errors and 5xx responses. Can be configured to retry on specific status codes
 // or to exclude specific codes from retry.
 //
+// For requests with bodies (POST, PUT, PATCH), the middleware handles body replay:
+// - If req.GetBody is set (automatic for strings.Reader, bytes.Buffer, bytes.Reader), it uses that
+// - If req.GetBody is nil and body buffering is disabled (default), requests won't be retried
+// - If body buffering is enabled with RetryBufferBodies(true), bodies up to maxBufferSize are buffered
+//
 // Default configuration:
 //   - 3 attempts
 //   - Initial delay: 100ms
 //   - Max delay: 30s
 //   - Exponential backoff
 //   - 10% jitter
 //   - Retries on 5xx status codes
+//   - Body buffering disabled (preserves streaming, no retries for bodies without GetBody)
 type RetryMiddleware struct {
-	next         http.RoundTripper
-	attempts     int
-	initialDelay time.Duration
-	maxDelay     time.Duration
-	backoff      BackoffType
-	jitterFactor float64
-	retryCodes   []int
-	excludeCodes []int
+	next          http.RoundTripper
+	attempts      int
+	initialDelay  time.Duration
+	maxDelay      time.Duration
+	backoff       BackoffType
+	jitterFactor  float64
+	retryCodes    []int
+	excludeCodes  []int
+	bufferBodies  bool
+	maxBufferSize int64
 }
 
 // Retry creates retry middleware with provided options
 func Retry(attempts int, initialDelay time.Duration, opts ...RetryOption) RoundTripperHandler {
 	return func(next http.RoundTripper) http.RoundTripper {
 		r := &RetryMiddleware{
-			next:         next,
-			attempts:     attempts,
-			initialDelay: initialDelay,
-			maxDelay:     30 * time.Second,
-			backoff:      BackoffExponential,
-			jitterFactor: 0.1,
+			next:          next,
+			attempts:      attempts,
+			initialDelay:  initialDelay,
+			maxDelay:      30 * time.Second,
+			backoff:       BackoffExponential,
+			jitterFactor:  0.1,
+			bufferBodies:  false, // disabled by default to preserve streaming; when enabled, reads entire body into memory
+			maxBufferSize: 10 * 1024 * 1024, // 10MB limit when buffering enabled
 		}
 
 		for _, opt := range opts {
@@ -69,10 +81,28 @@ func Retry(attempts int, initialDelay time.Duration, opts ...RetryOption) RoundT
 
 // RoundTrip implements http.RoundTripper
 func (r *RetryMiddleware) RoundTrip(req *http.Request) (*http.Response, error) {
+	// determine effective attempts based on body handling
+	attempts := r.attempts
+	hasBody := req.Body != nil && req.Body != http.NoBody
+
+	// prepare body for retries if needed
+	if hasBody && req.GetBody == nil && r.attempts > 1 {
+		if r.bufferBodies {
+			// try to buffer body for retries
+			if err := r.bufferRequestBody(req); err != nil {
+				// buffering failed or body too large
+				return nil, err
+			}
+		} else {
+			// buffering disabled - can't retry with body
+			attempts = 1
+		}
+	}
+
 	var lastResponse *http.Response
 	var lastError error
 
-	for attempt := 0; attempt < r.attempts; attempt++ {
+	for attempt := 0; attempt < attempts; attempt++ {
 		if req.Context().Err() != nil {
 			return nil, req.Context().Err()
 		}
@@ -84,6 +114,15 @@ func (r *RetryMiddleware) RoundTrip(req *http.Request) (*http.Response, error) {
 				return nil, req.Context().Err()
 			case <-time.After(delay):
 			}
+
+			// reset body for retry
+			if req.GetBody != nil {
+				newBody, err := req.GetBody()
+				if err != nil {
+					return nil, fmt.Errorf("retry: failed to get new request body: %w", err)
+				}
+				req.Body = newBody
+			}
 		}
 
 		resp, err := r.next.RoundTrip(req)
@@ -101,11 +140,36 @@ func (r *RetryMiddleware) RoundTrip(req *http.Request) (*http.Response, error) {
 	}
 
 	if lastError != nil {
-		return lastResponse, fmt.Errorf("retry: transport error after %d attempts: %w", r.attempts, lastError)
+		return lastResponse, fmt.Errorf("retry: transport error after %d attempts: %w", attempts, lastError)
 	}
 	return lastResponse, nil
 }
 
+// bufferRequestBody attempts to buffer the request body for retries
+// this consumes the original body - returns error if body is too large
+func (r *RetryMiddleware) bufferRequestBody(req *http.Request) error {
+	// read entire body (with limit for safety)
+	bodyBytes, err := io.ReadAll(io.LimitReader(req.Body, r.maxBufferSize+1))
+	if err != nil {
+		return fmt.Errorf("retry: failed to read request body: %w", err)
+	}
+	_ = req.Body.Close()
+
+	// check if body exceeds limit
+	if int64(len(bodyBytes)) > r.maxBufferSize {
+		return fmt.Errorf("retry: request body too large (%d bytes exceeds %d byte limit) - cannot retry",
+			len(bodyBytes), r.maxBufferSize)
+	}
+
+	// set up body and GetBody for retries
+	req.Body = io.NopCloser(bytes.NewReader(bodyBytes))
+	req.GetBody = func() (io.ReadCloser, error) {
+		return io.NopCloser(bytes.NewReader(bodyBytes)), nil
+	}
+
+	return nil
+}
+
 func (r *RetryMiddleware) calcDelay(attempt int) time.Duration {
 	if attempt == 0 {
 		return 0
@@ -192,3 +256,17 @@ func RetryExcludeCodes(codes ...int) RetryOption {
 		r.excludeCodes = codes
 	}
 }
+
+// RetryBufferBodies enables or disables automatic body buffering for retries
+func RetryBufferBodies(enabled bool) RetryOption {
+	return func(r *RetryMiddleware) {
+		r.bufferBodies = enabled
+	}
+}
+
+// RetryMaxBufferSize sets the maximum size of request bodies that will be buffered
+func RetryMaxBufferSize(size int64) RetryOption {
+	return func(r *RetryMiddleware) {
+		r.maxBufferSize = size
+	}
+}