add backoff

Author: kamushadenes

go.mod

@@ -4,6 +4,7 @@ go 1.21.1
 
 require (
 	github.com/aws/aws-sdk-go v1.53.19
+	github.com/cenkalti/backoff/v4 v4.3.0
 	github.com/google/go-containerregistry v0.19.1
 	github.com/prometheus/client_golang v1.19.1
 	github.com/rs/zerolog v1.33.0

go.sum

@@ -3,6 +3,8 @@ github.com/aws/aws-sdk-go v1.53.19 h1:WEuWc918RXlIaPCyU11F7hH9H1ItK+8m2c/uoQNRUo
 github.com/aws/aws-sdk-go v1.53.19/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
+github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/containerd/stargz-snapshotter/estargz v0.14.3 h1:OqlDCK3ZVUO6C3B/5FSkDwbkEETK84kQgEeFwDC+62k=

internal/sync/images.go

@@ -3,11 +3,13 @@ package sync
 import (
 	"context"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/Altinity/docker-sync/config"
 	"github.com/Altinity/docker-sync/internal/telemetry"
 	"github.com/Altinity/docker-sync/structs"
+	"github.com/cenkalti/backoff/v4"
 	"github.com/rs/zerolog/log"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/metric"
@@ -38,8 +40,23 @@ func SyncImage(ctx context.Context, image *structs.Image) error {
 			}
 		}
 
-		if serr := SyncTag(image, tag, pullAuthName, pullAuth); serr != nil {
-			errs := multierr.Errors(serr)
+		if err := backoff.Retry(func() error {
+			if err := SyncTag(image, tag, pullAuthName, pullAuth); err != nil {
+				if strings.Contains(err.Error(), "HAP429") {
+					log.Warn().
+						Str("source", image.Source).
+						Msg("Rate limited by registry, backing off")
+					return err
+				}
+
+				return backoff.Permanent(err)
+			}
+
+			return nil
+		}, backoff.NewExponentialBackOff(
+			backoff.WithInitialInterval(1*time.Minute),
+		)); err != nil {
+			errs := multierr.Errors(err)
 			if len(errs) > 0 {
 				telemetry.Errors.Add(ctx, int64(len(errs)),
 					metric.WithAttributes(
@@ -56,10 +73,10 @@ func SyncImage(ctx context.Context, image *structs.Image) error {
 				log.Error().
 					Errs("errors", errs).
 					Msg("Failed to sync tag")
-			}
 
-			merr = multierr.Append(merr, serr)
-			continue
+				merr = multierr.Append(merr, err)
+				continue
+			}
 		}
 		lastSyncMap[k] = time.Now()
 	}