[#2280] Ensure pods are ready before attempting to retrieve container

info on startup

Author: ryanemerson

pkg/reconcile/pipeline/infinispan/context/context.go

@@ -104,6 +104,9 @@ func (c *contextImpl) InfinispanClient() (api.Infinispan, error) {
 			break
 		}
 	}
+	if pod == "" {
+		return nil, fmt.Errorf("unable to create Infinispan client, no ready pods exist")
+	}
 	c.ispnClient = c.InfinispanClientForPod(pod)
 	return c.ispnClient, nil
 }

pkg/reconcile/pipeline/infinispan/handler/manage/upgrades.go

@@ -355,16 +355,22 @@ func EnableRebalanceAfterScaleUp(i *ispnv1.Infinispan, ctx pipeline.Context) {
 			return
 		}
 
+		if podList, err := ctx.InfinispanPods(); err != nil {
+			ctx.Requeue(err)
+			return
+		} else if len(podList.Items) != int(i.Spec.Replicas) {
+			ctx.Log().Info("Waiting on cluster pods to be provisioned", "pods", len(podList.Items), "spec.replicas", i.Spec.Replicas)
+			ctx.RequeueAfter(consts.DefaultWaitClusterPodsNotReady, nil)
+			return
+		}
+
 		ispnClient, err := ctx.InfinispanClient()
 		if err != nil {
-			ctx.Requeue(err)
+			ctx.Log().Info("Waiting on at least one ready pod", "err", err)
+			ctx.RequeueAfter(consts.DefaultWaitClusterPodsNotReady, nil)
 			return
 		}
 
-		// TODO why is this failing in TestOperandUpgrade for 14.0.x servers?
-		/*
-			2025-05-16T17:11:58.093+0100	ERROR	Reconciler error	{"controller": "infinispan", "controllerGroup": "infinispan.org", "controllerKind": "Infinispan", "infinispan": {"name":"test-operand-upgrade","namespace":"namespace-for-testing"}, "namespace": "namespace-for-testing", "name": "test-operand-upgrade", "reconcileID": "eda96c7e-fd83-43cc-b903-4a623e4e2785", "error": "unable to retrieve cluster information on scale up: unexpected error getting cache manager info: stderr: , err: the server does not allow this method on the requested resource"}
-		*/
 		if info, err := ispnClient.Container().Info(); err != nil {
 			ctx.Requeue(fmt.Errorf("unable to retrieve cluster information on scale up: %w", err))
 			return