K8SPSMDB-1265: clean up basmdb-backup and use kubectl wait for wait_restore (#1911)

Use kubectl wait instead of regular loop in `wait_restore()`
Add retry for `demand-backup-sharded` test backup presence in minio storage
Delete backups during test cleanup before removing finalizers from objects.

Author: eleo007

e2e-tests/demand-backup-incremental-sharded/run

@@ -56,7 +56,7 @@ run_recovery_check() {
 	fi
 	echo
 
-	wait_cluster_consistency ${cluster} 42
+	wait_cluster_consistency ${cluster} 60
 	wait_for_pbm_operations ${cluster}
 
 	if [[ $base == true ]]; then

e2e-tests/demand-backup-sharded/run

@@ -148,10 +148,18 @@ fi
 
 desc 'check backup and restore -- minio'
 backup_dest_minio=$(get_backup_dest "$backup_name_minio")
-kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
+retry=0
+until kubectl_bin run -i --rm aws-cli --image=perconalab/awscli --restart=Never -- \
 	/usr/bin/env AWS_ACCESS_KEY_ID=some-access-key AWS_SECRET_ACCESS_KEY=some-secret-key AWS_DEFAULT_REGION=us-east-1 \
 	/usr/bin/aws --endpoint-url http://minio-service:9000 s3 ls "s3://${backup_dest_minio}/rs0/" \
-	| grep "myApp.test.gz"
+	| grep "myApp.test.gz"; do
+	sleep 1
+	let retry+=1
+	if [ $retry -ge 60 ]; then
+		echo "Max retry count $retry reached. Something went wrong with writing backup"
+		exit 1
+	fi
+done
 insert_data_mongos "100501" "myApp" "" "$custom_port"
 insert_data_mongos "100501" "myApp1" "" "$custom_port"
 insert_data_mongos "100501" "myApp2" "" "$custom_port"

e2e-tests/functions

@@ -233,14 +233,15 @@ wait_backup() {
 		echo -n .
 		let retry+=1
 		current_status=$(kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.state}')
-		if [[ $retry -ge 360 || ${current_status} == 'error' ]]; then
+		if [[ $retry -ge 600 || ${current_status} == 'error' ]]; then
 			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
 				| grep -v 'level=info' \
 				| grep -v 'level=debug' \
 				| grep -v 'Getting tasks for pod' \
 				| grep -v 'Getting pods from source' \
-				| tail -100
-			echo "Backup object psmdb-backup/${backup_name} is in ${current_state} state."
+				| tail -200
+			kubectl_bin get psmdb-backup
+			echo "Backup object psmdb-backup/${backup_name} is in ${current_status} state."
 			echo something went wrong with operator or kubernetes cluster
 			exit 1
 		fi
@@ -379,35 +380,54 @@ wait_restore() {
 	local ok_if_ready=${6:-0}
 
 	set +o xtrace
+	# We need to run wait till object is created, otherwise wait fails at once
+	echo -n "Waiting for the psmdb-restore/restore-$backup_name object to be created"
+	retry_object=0
+	until kubectl_bin get psmdb-restore restore-$backup_name >/dev/null 2>&1; do
+		echo -n .
+		let retry_object+=1
+		if [[ ${retry_object} -ge 60 ]]; then
+			echo "psmdb-restore/restore-$backup_name object was not created."
+			exit 1
+		fi
+		sleep 1
+	done
+	echo "OK"
+
+	echo -n "Waiting psmdb-restore/restore-${backup_name} to reach state \"${target_state}\" "
 	retry=0
-	echo -n "waiting psmdb-restore/restore-${backup_name} to reach ${target_state} state"
-	local current_state=
-	until [[ ${current_state} == ${target_state} ]]; do
-		sleep 0.5
+	retry_count=$((wait_time / 60))
+	until kubectl wait psmdb-restore restore-${backup_name} --for=jsonpath='{.status.state}'=${target_state} --timeout=60s >/dev/null 2>&1; do
 		echo -n .
 		let retry+=1
 		current_state=$(kubectl_bin get psmdb-restore restore-$backup_name -o jsonpath='{.status.state}')
 		if [[ ${ok_if_ready} == 1 && ${current_state} == 'ready' ]]; then
 			echo "OK"
 			break
 		fi
-		if [[ $retry -ge $wait_time || ${current_state} == 'error' ]]; then
-			desc "operator logs:"
-			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) | tail -100
+		if [[ ${retry} -ge ${retry_count} || ${current_state} == 'error' ]]; then
+			kubectl_bin logs ${OPERATOR_NS:+-n $OPERATOR_NS} $(get_operator_pod) \
+				| grep -v 'level=info' \
+				| grep -v 'level=debug' \
+				| grep -v 'Getting tasks for pod' \
+				| grep -v 'Getting pods from source' \
+				| tail -100
 
 			if is_physical_backup ${backup_name}; then
 				collect_physical_restore_logs
 			fi
 
 			kubectl_bin get psmdb-restore restore-${backup_name} -o yaml
-			log "Restore object restore-${backup_name} is in ${current_state} state."
+
+			echo "Restore object restore-${backup_name} is in ${current_state} state."
+			echo something went wrong with operator or kubernetes cluster
 			exit 1
 		fi
 	done
 	echo "OK"
 	set_debug
 
-	if [ $wait_cluster_consistency -eq 1 ]; then
+	if [[ $wait_cluster_consistency -eq 1 ]]; then
 		wait_cluster_consistency "${cluster_name}"
 	fi
 }
@@ -1054,6 +1074,14 @@ delete_crd() {
 	kubectl_bin delete -f "${src_dir}/deploy/$rbac_yaml" --ignore-not-found || true
 }
 
+delete_backups() {
+	desc 'Delete psmdb-backup'
+	if [ $(kubectl_bin get psmdb-backup --no-headers | wc -l) != 0 ]; then
+		kubectl_bin get psmdb-backup
+		kubectl_bin delete psmdb-backup --all
+	fi
+}
+
 destroy() {
 	local namespace="$1"
 	local ignore_logs="${2:-true}"
@@ -1074,6 +1102,8 @@ destroy() {
 	#TODO: maybe will be enabled later
 	#diff $test_dir/compare/operator.log $tmp_dir/operator.log
 
+	delete_backups
+
 	delete_crd
 
 	destroy_cert_manager || true

e2e-tests/pitr-physical/run

@@ -78,6 +78,8 @@ main() {
 		sleep 10
 	done
 
+	sleep 10
+
 	check_recovery $backup_name_minio-2 date "${last_chunk}" "-2nd" "$cluster"
 
 	run_backup $backup_name_minio 3 physical