backoff wait after adding resync

Author: pooknull

e2e-tests/demand-backup-incremental-sharded/run

@@ -50,7 +50,7 @@ run_recovery_check() {
 	# we don't wait for cluster readiness here because the annotation gets removed then
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "3000"
 
-	sleep 5
+	sleep 15
 
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		log "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a incremental restore"

e2e-tests/demand-backup-incremental/run

@@ -50,7 +50,7 @@ run_recovery_check() {
 	# we don't wait for cluster readiness here because the annotation gets removed then
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "1800"
 
-	sleep 5
+	sleep 15
 
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		log "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a incremental restore"

e2e-tests/demand-backup-physical-sharded/run

@@ -37,7 +37,7 @@ run_recovery_check() {
 	# we don't wait for cluster readiness here because the annotation gets removed then
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "3000"
 
-	sleep 5
+	sleep 15
 
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"

e2e-tests/demand-backup-physical/run

@@ -37,7 +37,7 @@ run_recovery_check() {
 	# we don't wait for cluster readiness here because the annotation gets removed then
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "1800"
 
-	sleep 5
+	sleep 15
 
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"

pkg/controller/perconaservermongodbrestore/physical.go

@@ -400,7 +400,6 @@ func (r *ReconcilePerconaServerMongoDBRestore) finishPhysicalRestore(ctx context
 		return false, nil
 	}
 
-	resyncPresent := false
 	if err = retry.RetryOnConflict(retry.DefaultBackoff, func() error {
 		c := new(psmdbv1.PerconaServerMongoDB)
 		if err := r.client.Get(ctx, client.ObjectKeyFromObject(cluster), c); err != nil {
@@ -411,7 +410,6 @@ func (r *ReconcilePerconaServerMongoDBRestore) finishPhysicalRestore(ctx context
 			c.Annotations = make(map[string]string)
 		}
 		if c.Annotations[psmdbv1.AnnotationResyncPBM] == "true" {
-			resyncPresent = true
 			return nil
 		}
 
@@ -421,8 +419,27 @@ func (r *ReconcilePerconaServerMongoDBRestore) finishPhysicalRestore(ctx context
 	}); err != nil {
 		return false, errors.Wrapf(err, "annotate psmdb/%s for PBM resync", cluster.Name)
 	}
-	if !resyncPresent {
-		return false, nil
+
+	err = retry.OnError(retry.DefaultBackoff, func(err error) bool {
+		return err != nil
+	}, func() error {
+		c := new(psmdbv1.PerconaServerMongoDB)
+		if err := r.client.Get(ctx, client.ObjectKeyFromObject(cluster), c); err != nil {
+			return err
+		}
+
+		if c.Annotations == nil {
+			return errors.New("annotation wasn't added")
+		}
+
+		if v, ok := c.Annotations[psmdbv1.AnnotationResyncPBM]; !ok || v != "true" {
+			return errors.New("annotation wasn't added 2")
+		}
+
+		return nil
+	})
+	if err != nil {
+		return false, errors.Wrap(err, "failed to check if annotation was added")
 	}
 
 	if err := r.updateMongodSts(ctx, cluster, func(sts *appsv1.StatefulSet) error {