K8SPSMDB-1263: retry on error during running backup (#1838)

* K8SPSMDB-1263: retry on error during running backup

https://perconadev.atlassian.net/browse/K8SPSMDB-1263

* fix

* small improvements

---------

Co-authored-by: Viacheslav Sarzhan <slava.sarzhan@percona.com>

Author: pooknull

pkg/controller/perconaservermongodbbackup/backup.go

@@ -8,13 +8,15 @@ import (
 
 	"github.com/pkg/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	logf "sigs.k8s.io/controller-runtime/pkg/log"
 
 	pbmBackup "github.com/percona/percona-backup-mongodb/pbm/backup"
 	"github.com/percona/percona-backup-mongodb/pbm/ctrl"
 	"github.com/percona/percona-backup-mongodb/pbm/defs"
 	pbmErrors "github.com/percona/percona-backup-mongodb/pbm/errors"
+
 	api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
 	"github.com/percona/percona-server-mongodb-operator/pkg/psmdb/backup"
 )
@@ -25,6 +27,13 @@ const (
 	pbmStartingDeadlineErrMsg = "starting deadline exceeded"
 )
 
+var defaultBackoff = wait.Backoff{
+	Duration: 10 * time.Second,
+	Factor:   2.0,
+	Cap:      time.Minute * 5,
+	Steps:    6,
+}
+
 type Backup struct {
 	pbm  backup.PBM
 	spec api.BackupSpec
@@ -34,7 +43,7 @@ func (r *ReconcilePerconaServerMongoDBBackup) newBackup(ctx context.Context, clu
 	if cluster == nil {
 		return new(Backup), nil
 	}
-	cn, err := backup.NewPBM(ctx, r.client, cluster)
+	cn, err := r.newPBMFunc(ctx, r.client, cluster)
 	if err != nil {
 		return nil, errors.Wrap(err, "create pbm object")
 	}

pkg/controller/perconaservermongodbbackup/perconaservermongodbbackup_controller.go

@@ -26,6 +26,7 @@ import (
 	"github.com/percona/percona-backup-mongodb/pbm/storage"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/azure"
 	"github.com/percona/percona-backup-mongodb/pbm/storage/s3"
+
 	"github.com/percona/percona-server-mongodb-operator/clientcmd"
 	psmdbv1 "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1"
 	"github.com/percona/percona-server-mongodb-operator/pkg/naming"
@@ -169,9 +170,16 @@ func (r *ReconcilePerconaServerMongoDBBackup) Reconcile(ctx context.Context, req
 		}
 	}
 
-	bcp, err := r.newBackup(ctx, cluster)
-	if err != nil {
-		return rr, errors.Wrap(err, "create backup object")
+	var bcp *Backup
+	if err = retry.OnError(defaultBackoff, func(err error) bool { return err != nil }, func() error {
+		var err error
+		bcp, err = r.newBackup(ctx, cluster)
+		if err != nil {
+			return errors.Wrap(err, "create backup object")
+		}
+		return nil
+	}); err != nil {
+		return rr, err
 	}
 	defer bcp.Close(ctx)
 
@@ -209,26 +217,37 @@ func (r *ReconcilePerconaServerMongoDBBackup) reconcile(
 		return status, errors.Wrap(err, "failed to run backup")
 	}
 
-	cjobs, err := backup.HasActiveJobs(ctx, r.newPBMFunc, r.client, cluster, backup.NewBackupJob(cr.Name), backup.NotPITRLock)
-	if err != nil {
-		return status, errors.Wrap(err, "check for concurrent jobs")
-	}
-
-	if cjobs {
-		if cr.Status.State != psmdbv1.BackupStateWaiting {
-			log.Info("Waiting to finish another backup/restore.")
-		}
-		status.State = psmdbv1.BackupStateWaiting
-		return status, nil
-	}
-
-	if cr.Status.State == psmdbv1.BackupStateNew || cr.Status.State == psmdbv1.BackupStateWaiting {
+	switch cr.Status.State {
+	case psmdbv1.BackupStateNew, psmdbv1.BackupStateWaiting:
 		time.Sleep(10 * time.Second)
 		return bcp.Start(ctx, r.client, cluster, cr)
+	case psmdbv1.BackupStateRunning:
+	default:
+		cjobs, err := backup.HasActiveJobs(ctx, r.newPBMFunc, r.client, cluster, backup.NewBackupJob(cr.Name), backup.NotPITRLock)
+		if err != nil {
+			return status, errors.Wrap(err, "check for concurrent jobs")
+		}
+
+		if cjobs {
+			if cr.Status.State != psmdbv1.BackupStateWaiting {
+				log.Info("Waiting to finish another backup/restore.")
+			}
+			status.State = psmdbv1.BackupStateWaiting
+			return status, nil
+		}
 	}
 
 	time.Sleep(5 * time.Second)
-	return bcp.Status(ctx, cr)
+
+	err := retry.OnError(defaultBackoff, func(err error) bool { return err != nil }, func() error {
+		updatedStatus, err := bcp.Status(ctx, cr)
+		if err == nil {
+			status = updatedStatus
+		}
+		return err
+	})
+
+	return status, err
 }
 
 func (r *ReconcilePerconaServerMongoDBBackup) getPBMStorage(ctx context.Context, cluster *psmdbv1.PerconaServerMongoDB, cr *psmdbv1.PerconaServerMongoDBBackup) (storage.Storage, error) {