Make org.infinispan.LOCKS cache as AVAILABLE automatically

Author: ryanemerson

test/e2e/upgrade/common.go

@@ -259,7 +259,27 @@ func checkBatch(t *testing.T, infinispan *ispnv1.Infinispan) {
 	testKube.DeleteBatch(batch)
 }
 
-func assertNoDegradedCaches() {
+func locksCacheDegraded(operand version.Operand) bool {
+	if operand.UpstreamVersion.Major > 14 {
+		return false
+	}
+	podList := &corev1.PodList{}
+	tutils.ExpectNoError(
+		testKube.Kubernetes.ResourcesList(tutils.Namespace, map[string]string{"app": "infinispan-pod"}, podList, ctx),
+	)
+
+	for _, pod := range podList.Items {
+		log, err := testKube.Kubernetes.Logs(provision.InfinispanContainer, pod.GetName(), pod.GetNamespace(), false, ctx)
+		tutils.ExpectNoError(err)
+		if strings.Contains(log, "DEGRADED_MODE") {
+			return true
+		}
+	}
+	return false
+}
+
+// healDegradedLocksCache forces the org.infinispan.LOCKS cache to AVAILABLE so that subsequent upgrades can proceed
+func healDegradedLocksCache(operand version.Operand, client tutils.HTTPClient) {
 	podList := &corev1.PodList{}
 	tutils.ExpectNoError(
 		testKube.Kubernetes.ResourcesList(tutils.Namespace, map[string]string{"app": "infinispan-pod"}, podList, ctx),
@@ -269,7 +289,8 @@ func assertNoDegradedCaches() {
 		log, err := testKube.Kubernetes.Logs(provision.InfinispanContainer, pod.GetName(), pod.GetNamespace(), false, ctx)
 		tutils.ExpectNoError(err)
 		if strings.Contains(log, "DEGRADED_MODE") {
-			panic("Detected a cache in DEGRADED_MODE, unable to continue with test")
+			tutils.Log().Infof("Setting org.infinispan.LOCKS cache to AVAILABLE for Operand %s", operand)
+			tutils.NewCacheHelper("org.infinispan.LOCKS", client).Available(true)
 		}
 	}
 }

test/e2e/upgrade/upgrade_operands_test.go

@@ -41,11 +41,12 @@ func TestOperandUpgrades(t *testing.T) {
 
 	// Create the Infinispan CR
 	replicas := 2
+	startingOperand := versionManager.Operands[startingOperandIdx]
 	ispn := tutils.DefaultSpec(t, testKube, func(i *ispnv1.Infinispan) {
 		i.Spec.Replicas = int32(replicas)
 		i.Spec.Service.Container.EphemeralStorage = false
 		i.Spec.Logging.Categories["org.infinispan.topology"] = ispnv1.LoggingLevelTrace
-		i.Spec.Version = versionManager.Operands[startingOperandIdx].Ref()
+		i.Spec.Version = startingOperand.Ref()
 	})
 
 	log.Infof("Testing upgrades from Operand '%s' to '%s'", ispn.Spec.Version, versionManager.Latest().Ref())
@@ -56,15 +57,17 @@ func TestOperandUpgrades(t *testing.T) {
 	numEntries := 100
 	client := tutils.HTTPClientForClusterWithVersionManager(ispn, testKube, versionManager)
 
+	if op := *startingOperand; locksCacheDegraded(op) {
+		healDegradedLocksCache(op, client)
+	}
+
 	// Add a persistent cache with data to ensure contents can be read after upgrade(s)
 	createAndPopulatePersistentCache(persistentCacheName, numEntries, client)
 
 	// Add a volatile cache with data to ensure contents can be backed up and then restored after upgrade(s)
 	createAndPopulateVolatileCache(volatileCacheName, numEntries, client)
 
-	assertNoDegradedCaches()
 	backup := createBackupAndWaitToSucceed(ispn.Name, t)
-
 	skippedOperands := tutils.OperandSkipSet()
 	for _, operand := range versionManager.Operands[startingOperandIdx:] {
 		// Skip an Operand in the upgrade graph if there's a known issue
@@ -75,6 +78,10 @@ func TestOperandUpgrades(t *testing.T) {
 		log.Debugf("Next Operand %s", operand.Ref())
 
 		ispn = testKube.WaitForInfinispanConditionWithTimeout(ispn.Name, tutils.Namespace, ispnv1.ConditionWellFormed, conditionTimeout)
+		if op := *startingOperand; locksCacheDegraded(op) {
+			healDegradedLocksCache(op, client)
+		}
+
 		tutils.ExpectNoError(
 			testKube.UpdateInfinispan(ispn, func() {
 				ispn.Spec.Version = operand.Ref()
@@ -89,8 +96,6 @@ func TestOperandUpgrades(t *testing.T) {
 				i.Status.Operand.Phase == ispnv1.OperandPhaseRunning
 		})
 		log.Info("Upgrade complete")
-		assertOperandImage(operand.Image, ispn)
-		assertNoDegradedCaches()
 
 		// Ensure that persistent cache entries have survived the upgrade(s)
 		// Refresh the hostAddr and client as the url will change if NodePort is used.

test/e2e/upgrade/upgrade_test.go

@@ -42,11 +42,18 @@ func TestUpgrade(t *testing.T) {
 
 	testKube.CreateInfinispan(spec, tutils.Namespace)
 	testKube.WaitForInfinispanPods(replicas, tutils.SinglePodTimeout, spec.Name, tutils.Namespace)
-	testKube.WaitForInfinispanConditionWithTimeout(spec.Name, spec.Namespace, ispnv1.ConditionWellFormed, conditionTimeout)
-	assertNoDegradedCaches()
+	spec = testKube.WaitForInfinispanConditionWithTimeout(spec.Name, spec.Namespace, ispnv1.ConditionWellFormed, conditionTimeout)
+
+	versionManager := testKube.VersionManagerFromCSV(sub)
+	operand, err := versionManager.WithRef(spec.Spec.Version)
+	tutils.ExpectNoError(err)
 
 	numEntries := 100
-	client := tutils.HTTPClientForClusterWithVersionManager(spec, testKube, testKube.VersionManagerFromCSV(sub))
+	client := tutils.HTTPClientForClusterWithVersionManager(spec, testKube, versionManager)
+
+	if locksCacheDegraded(operand) {
+		healDegradedLocksCache(operand, client)
+	}
 
 	// Add a persistent cache with data to ensure contents can be read after upgrade(s)
 	createAndPopulatePersistentCache(persistentCacheName, numEntries, client)
@@ -77,7 +84,7 @@ func TestUpgrade(t *testing.T) {
 		// https://github.com/infinispan/infinispan-operator/issues/1719
 		time.Sleep(time.Minute)
 
-		versionManager := testKube.VersionManagerFromCSV(sub)
+		versionManager = testKube.VersionManagerFromCSV(sub)
 		if ispnPreUpgrade.Spec.Version == "" {
 			relatedImageJdk := testKube.InstalledCSVEnv("RELATED_IMAGE_OPENJDK", sub)
 			if relatedImageJdk != "" {
@@ -118,6 +125,13 @@ func TestUpgrade(t *testing.T) {
 				continue
 			}
 
+			client = tutils.HTTPClientForClusterWithVersionManager(spec, testKube, versionManager)
+			op, err := versionManager.WithRef(ispnPreUpgrade.Spec.Version)
+			tutils.ExpectNoError(err)
+			if locksCacheDegraded(op) {
+				healDegradedLocksCache(op, client)
+			}
+
 			tutils.ExpectNoError(
 				testKube.UpdateInfinispan(ispn, func() {
 					ispn.Spec.Version = latestOperand.Ref()
@@ -142,9 +156,12 @@ func TestUpgrade(t *testing.T) {
 			// Ensure that persistent cache entries have survived the upgrade(s)
 			// Refresh the hostAddr and client as the url will change if NodePort is used.
 			client = tutils.HTTPClientForClusterWithVersionManager(spec, testKube, versionManager)
-			assertNoDegradedCaches()
 			tutils.NewCacheHelper(persistentCacheName, client).AssertSize(numEntries)
 
+			if locksCacheDegraded(latestOperand) {
+				healDegradedLocksCache(latestOperand, client)
+			}
+
 			// Restore the backup and ensure that the cache exists with the expected number of entries
 			restoreName := "upgrade-restore-" + strings.ReplaceAll(strings.TrimLeft(sub.Status.CurrentCSV, olm.SubName+".v"), ".", "-")
 			if restore, err := createRestoreAndWaitToSucceed(restoreName, backup, t); err != nil {
@@ -163,7 +180,7 @@ func TestUpgrade(t *testing.T) {
 	checkBatch(t, spec)
 
 	// Kill the first pod to ensure that the cluster can recover from failover after upgrade
-	err := testKube.Kubernetes.Client.Delete(ctx, &corev1.Pod{
+	err = testKube.Kubernetes.Client.Delete(ctx, &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      spec.Name + "-0",
 			Namespace: tutils.Namespace,
@@ -174,7 +191,7 @@ func TestUpgrade(t *testing.T) {
 	testKube.WaitForInfinispanConditionWithTimeout(spec.Name, tutils.Namespace, ispnv1.ConditionWellFormed, conditionTimeout)
 
 	// Ensure that persistent cache entries still contain the expected numEntries
-	versionManager := testKube.VersionManagerFromCSV(sub)
+	versionManager = testKube.VersionManagerFromCSV(sub)
 	client = tutils.HTTPClientForClusterWithVersionManager(spec, testKube, versionManager)
 	tutils.NewCacheHelper(persistentCacheName, client).AssertSize(numEntries)
 }