ST: Make several systemtests checks more resilient (#11373)

Signed-off-by: Jakub Stejskal <xstejs24@gmail.com>

Author: Frawless

systemtest/src/main/java/io/strimzi/systemtest/resources/NamespaceManager.java

@@ -240,7 +240,7 @@ public void deleteAllNamespacesFromSet() {
      * @param namespaceName name of the Namespace that should be deleted
      */
     private void waitForNamespaceDeletion(String namespaceName) {
-        TestUtils.waitFor("Namespace: " + namespaceName + "to be deleted", TestConstants.POLL_INTERVAL_FOR_RESOURCE_DELETION, DELETION_TIMEOUT, () -> {
+        TestUtils.waitFor("Namespace: " + namespaceName + " to be deleted", TestConstants.POLL_INTERVAL_FOR_RESOURCE_DELETION, DELETION_TIMEOUT, () -> {
             Namespace namespace = kubeClient().getNamespace(namespaceName);
 
             if (namespace == null) {
@@ -249,6 +249,9 @@ private void waitForNamespaceDeletion(String namespaceName) {
                 LOGGER.debug("There are KafkaTopics with finalizers remaining in Namespace: {}, going to set those finalizers to null", namespaceName);
                 KafkaTopicUtils.setFinalizersInAllTopicsToNull(namespaceName);
             }
+            // Try to delete it once-again in case something is stuck
+            deleteNamespace(namespaceName);
+
             return false;
         });
     }

systemtest/src/main/java/io/strimzi/systemtest/resources/operator/specific/HelmResource.java

@@ -90,7 +90,7 @@ private void clusterOperator(long operationTimeout, long reconciliationInterval,
         values.put("kafkaBridge.image.tag", bridgeTag);
 
         // Additional config
-        values.put("image.imagePullPolicy", Environment.OPERATOR_IMAGE_PULL_POLICY);
+        values.put("image.imagePullPolicy", Environment.COMPONENTS_IMAGE_PULL_POLICY);
         values.put("resources.requests.memory", TestConstants.CO_REQUESTS_MEMORY);
         values.put("resources.requests.cpu", TestConstants.CO_REQUESTS_CPU);
         values.put("resources.limits.memory", TestConstants.CO_LIMITS_MEMORY);

systemtest/src/main/java/io/strimzi/systemtest/utils/ClientUtils.java

@@ -321,8 +321,8 @@ public static void waitForClientNotContainsMessage(String namespaceName, String
     }
 
     private static long timeoutForClientFinishJob(int messagesCount) {
-        // need to add at least 2minutes for finishing the job
-        return (long) messagesCount * 1000 + Duration.ofMinutes(2).toMillis();
+        // need to add at least 3 minutes for finishing the job (due to slow environments)
+        return (long) messagesCount * 1000 + Duration.ofMinutes(3).toMillis();
     }
 
     /**

systemtest/src/test/java/io/strimzi/systemtest/kafka/KafkaST.java

@@ -413,7 +413,15 @@ void testKafkaJBODDeleteClaimsTrueFalse() {
 
         Map<String, String> brokerPods = PodUtils.podSnapshot(testStorage.getNamespaceName(), testStorage.getBrokerSelector());
         // kafka cluster already deployed
-        verifyVolumeNamesAndLabels(testStorage.getNamespaceName(), testStorage.getClusterName(), testStorage.getBrokerComponentName(), kafkaReplicas, 2, diskSizeGi);
+        TestUtils.waitFor("expected labels on PVCs", TestConstants.GLOBAL_POLL_INTERVAL_5_SECS, TestConstants.GLOBAL_STATUS_TIMEOUT, () -> {
+            try {
+                verifyVolumeNamesAndLabels(testStorage.getNamespaceName(), testStorage.getClusterName(), testStorage.getBrokerComponentName(), kafkaReplicas, 2, diskSizeGi);
+                return true;
+            } catch (AssertionError ex) {
+                LOGGER.info("Some of the expected labels are not in place, rerunning the check...");
+                return false;
+            }
+        });
 
         //change value of first PVC to delete its claim once Kafka is deleted.
         LOGGER.info("Update Volume with id=0 in Kafka CR by setting 'Delete Claim' property to false");
@@ -841,8 +849,16 @@ void testMessagesAndConsumerOffsetFilesOnDisk() {
         String brokerPodName = kubeClient().listPods(testStorage.getNamespaceName(), testStorage.getBrokerSelector()).get(0).getMetadata().getName();
 
         TestUtils.waitFor("KafkaTopic creation inside Kafka Pod", TestConstants.GLOBAL_POLL_INTERVAL, TestConstants.GLOBAL_TIMEOUT,
-            () -> cmdKubeClient(testStorage.getNamespaceName()).execInPod(brokerPodName, "/bin/bash",
-                "-c", "cd /var/lib/kafka/data/kafka-log0; ls -1").out().contains(testStorage.getTopicName()));
+            () -> {
+                String output = cmdKubeClient(testStorage.getNamespaceName()).execInPod(brokerPodName, "/bin/bash",
+                    "-c", "cd /var/lib/kafka/data/kafka-log0; ls -1").out();
+                if (output.contains(testStorage.getTopicName())) {
+                    return true;
+                } else {
+                    LOGGER.debug("{} should be in {}, trying again...", testStorage.getTopicName(), output);
+                    return false;
+                }
+            });
 
         String topicDirNameInPod = cmdKubeClient(testStorage.getNamespaceName()).execInPod(brokerPodName, "/bin/bash",
             "-c", "cd /var/lib/kafka/data/kafka-log0; ls -1 | sed -n '/" + testStorage.getTopicName() + "/p'").out();
@@ -1323,7 +1339,7 @@ void verifyVolumeNamesAndLabels(String namespaceName, String clusterName, String
             .forEach(volume -> {
                 String volumeName = volume.getMetadata().getName();
                 pvcs.add(volumeName);
-                LOGGER.info("Checking labels for volume:" + volumeName);
+                LOGGER.info("Checking labels for volume: " + volumeName);
                 assertThat(volume.getMetadata().getLabels().get(Labels.STRIMZI_CLUSTER_LABEL), is(clusterName));
                 assertThat(volume.getMetadata().getLabels().get(Labels.STRIMZI_KIND_LABEL), is(Kafka.RESOURCE_KIND));
                 assertThat(volume.getMetadata().getLabels().get(Labels.STRIMZI_NAME_LABEL), is(clusterName.concat("-kafka")));

systemtest/src/test/java/io/strimzi/systemtest/upgrade/KRaftKafkaUpgradeDowngradeST.java

@@ -23,6 +23,7 @@
 import io.strimzi.systemtest.utils.TestKafkaVersion;
 import io.strimzi.systemtest.utils.kafkaUtils.KafkaUtils;
 import io.strimzi.systemtest.utils.kubeUtils.objects.PodUtils;
+import io.strimzi.test.TestUtils;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
 import org.junit.jupiter.api.BeforeAll;
@@ -249,13 +250,17 @@ void runVersionChange(TestStorage testStorage, TestKafkaVersion initialVersion,
 
         if (!isUpgrade) {
             LOGGER.info("Verifying that metadataVersion attribute updated correctly to version {}", initMetadataVersion);
-            assertThat(Crds.kafkaOperation(kubeClient().getClient()).inNamespace(testStorage.getNamespaceName()).withName(CLUSTER_NAME)
-                .get().getStatus().getKafkaMetadataVersion().contains(initMetadataVersion), is(true));
+            TestUtils.waitFor(String.format("metadataVersion attribute updated correctly to version %s", initMetadataVersion),
+                TestConstants.GLOBAL_POLL_INTERVAL, TestConstants.GLOBAL_STATUS_TIMEOUT,
+                () -> Crds.kafkaOperation(kubeClient().getClient()).inNamespace(testStorage.getNamespaceName()).withName(CLUSTER_NAME)
+                    .get().getStatus().getKafkaMetadataVersion().contains(initMetadataVersion));
         } else {
             if (currentMetadataVersion != null) {
                 LOGGER.info("Verifying that metadataVersion attribute updated correctly to version {}", newVersion.metadataVersion());
-                assertThat(Crds.kafkaOperation(kubeClient().getClient()).inNamespace(testStorage.getNamespaceName()).withName(CLUSTER_NAME)
-                    .get().getStatus().getKafkaMetadataVersion().contains(newVersion.metadataVersion()), is(true));
+                TestUtils.waitFor(String.format("metadataVersion attribute updated correctly to version %s", newVersion.metadataVersion()),
+                    TestConstants.GLOBAL_POLL_INTERVAL, TestConstants.GLOBAL_STATUS_TIMEOUT,
+                    () -> Crds.kafkaOperation(kubeClient().getClient()).inNamespace(testStorage.getNamespaceName()).withName(CLUSTER_NAME)
+                        .get().getStatus().getKafkaMetadataVersion().contains(newVersion.metadataVersion()));
             }
         }