(feat) Report suspension reasons

api/v1beta1/clustersummary_types.go

@@ -168,6 +168,16 @@ type ClusterSummaryStatus struct {
 	// instance itself, *excluding* errors related to the deployment of individual features.
 	// +optional
 	FailureMessage *string `json:"failureMessage,omitempty"`
+
+	// ReconciliationSuspended indicates whether the reconciliation loop for this
+	// ClusterSummary is currently paused due to an external action (e.g., a user annotation).
+	// When true, the status will not be updated unless the pause is lifted.
+	// +optional
+	ReconciliationSuspended bool `json:"reconciliationSuspended,omitempty"`
+
+	// SuspensionReason provides a brief explanation of why the reconciliation is suspended.
+	// +optional
+	SuspensionReason *string `json:"suspensionReason,omitempty"`
 }
 
 //nolint: lll // marker

api/v1beta1/status.go

@@ -41,4 +41,14 @@ type Status struct {
 	// DependenciesHash is a hash representing the set of clusters where this ClusterProfile
 	// must be deployed, based on the combined configuration of its dependencies.
 	DependenciesHash []byte `json:"dependenciesHash,omitempty"`
+
+	// ReconciliationSuspended indicates whether the reconciliation loop for this
+	// ClusterSummary is currently paused due to an external action (e.g., a user annotation).
+	// When true, the status will not be updated unless the pause is lifted.
+	// +optional
+	ReconciliationSuspended bool `json:"reconciliationSuspended,omitempty"`
+
+	// SuspensionReason provides a brief explanation of why the reconciliation is suspended.
+	// +optional
+	SuspensionReason *string `json:"suspensionReason,omitempty"`
 }

config/crd/bases/config.projectsveltos.io_clusterprofiles.yaml

@@ -1230,6 +1230,16 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              reconciliationSuspended:
+                description: |-
+                  ReconciliationSuspended indicates whether the reconciliation loop for this
+                  ClusterSummary is currently paused due to an external action (e.g., a user annotation).
+                  When true, the status will not be updated unless the pause is lifted.
+                type: boolean
+              suspensionReason:
+                description: SuspensionReason provides a brief explanation of why
+                  the reconciliation is suspended.
+                type: string
               updatedClusters:
                 description: |-
                   UpdatedClusters contains information all the cluster currently matching

config/crd/bases/config.projectsveltos.io_clustersummaries.yaml

@@ -1376,6 +1376,16 @@ spec:
                   If not set, reconciliations will happen as usual.
                 format: date-time
                 type: string
+              reconciliationSuspended:
+                description: |-
+                  ReconciliationSuspended indicates whether the reconciliation loop for this
+                  ClusterSummary is currently paused due to an external action (e.g., a user annotation).
+                  When true, the status will not be updated unless the pause is lifted.
+                type: boolean
+              suspensionReason:
+                description: SuspensionReason provides a brief explanation of why
+                  the reconciliation is suspended.
+                type: string
             type: object
         type: object
     served: true

config/crd/bases/config.projectsveltos.io_profiles.yaml

@@ -1230,6 +1230,16 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              reconciliationSuspended:
+                description: |-
+                  ReconciliationSuspended indicates whether the reconciliation loop for this
+                  ClusterSummary is currently paused due to an external action (e.g., a user annotation).
+                  When true, the status will not be updated unless the pause is lifted.
+                type: boolean
+              suspensionReason:
+                description: SuspensionReason provides a brief explanation of why
+                  the reconciliation is suspended.
+                type: string
               updatedClusters:
                 description: |-
                   UpdatedClusters contains information all the cluster currently matching

controllers/clusterprofile_controller.go

@@ -124,9 +124,15 @@ func (r *ClusterProfileReconciler) Reconcile(ctx context.Context, req ctrl.Reque
 
 	if isProfilePaused(profileScope) {
 		logger.V(logs.LogInfo).Info("profile is paused. Skip reconciliation")
+		profileScope.GetStatus().ReconciliationSuspended = true
+		suspensionReason := "ClusterProfile is paused"
+		profileScope.GetStatus().SuspensionReason = &suspensionReason
 		return reconcile.Result{}, nil
 	}
 
+	profileScope.GetStatus().ReconciliationSuspended = false
+	profileScope.GetStatus().SuspensionReason = nil
+
 	// Handle deleted clusterProfile
 	if !clusterProfile.DeletionTimestamp.IsZero() {
 		return r.reconcileDelete(ctx, profileScope), nil

controllers/clustersummary_controller.go

@@ -89,6 +89,10 @@ const (
 	AgentSendUpdatesNoGateway
 )
 
+const (
+	clusterPausedMessage = "Cluster is paused"
+)
+
 // ClusterSummaryReconciler reconciles a ClusterSummary object
 type ClusterSummaryReconciler struct {
 	*rest.Config
@@ -270,9 +274,15 @@ func (r *ClusterSummaryReconciler) reconcileDelete(
 		}
 		if paused {
 			logger.V(logs.LogInfo).Info("cluster is paused. Do nothing.")
+			clusterSummaryScope.ClusterSummary.Status.ReconciliationSuspended = true
+			suspensionReason := clusterPausedMessage
+			clusterSummaryScope.ClusterSummary.Status.SuspensionReason = &suspensionReason
 			return reconcile.Result{}, nil
 		}
 
+		clusterSummaryScope.ClusterSummary.Status.ReconciliationSuspended = false
+		clusterSummaryScope.ClusterSummary.Status.SuspensionReason = nil
+
 		if !isDeleted {
 			// if cluster is marked for deletion do not try to remove ResourceSummaries.
 			// those are only deployed in the managed cluster so no need to cleanup on a deleted cluster
@@ -287,21 +297,7 @@ func (r *ClusterSummaryReconciler) reconcileDelete(
 		// in the management cluster and those need to be removed.
 		err = r.undeploy(ctx, clusterSummaryScope, logger)
 		if err != nil {
-			// In DryRun mode it is expected to always get an error back
-			if !clusterSummaryScope.IsDryRunSync() {
-				logger.V(logs.LogInfo).Error(err, "failed to undeploy")
-				return reconcile.Result{Requeue: true, RequeueAfter: deleteRequeueAfter}, nil
-			}
-
-			var nonRetriableError *configv1beta1.NonRetriableError
-			if errors.As(err, &nonRetriableError) {
-				return reconcile.Result{Requeue: true, RequeueAfter: deleteHandOverRequeueAfter}, nil
-			}
-
-			var templateError *configv1beta1.TemplateInstantiationError
-			if errors.As(err, &templateError) {
-				return reconcile.Result{Requeue: true, RequeueAfter: deleteHandOverRequeueAfter}, nil
-			}
+			return r.processUndeployError(clusterSummaryScope, err, logger)
 		}
 
 		if !r.canRemoveFinalizer(ctx, clusterSummaryScope, logger) {
@@ -319,12 +315,9 @@ func (r *ClusterSummaryReconciler) reconcileDelete(
 	}
 
 	// Cluster is not present anymore or cleanup succeeded
-	logger.V(logs.LogInfo).Info("Removing finalizer")
-	if controllerutil.ContainsFinalizer(clusterSummaryScope.ClusterSummary, configv1beta1.ClusterSummaryFinalizer) {
-		if finalizersUpdated := controllerutil.RemoveFinalizer(clusterSummaryScope.ClusterSummary,
-			configv1beta1.ClusterSummaryFinalizer); !finalizersUpdated {
-			return reconcile.Result{}, fmt.Errorf("failed to remove finalizer")
-		}
+	err = r.removeFinalizer(clusterSummaryScope, logger)
+	if err != nil {
+		return reconcile.Result{}, fmt.Errorf("failed to remove finalizer")
 	}
 
 	if err := r.deleteChartMap(ctx, clusterSummaryScope, logger); err != nil {
@@ -379,9 +372,15 @@ func (r *ClusterSummaryReconciler) reconcileNormal(ctx context.Context,
 	}
 	if paused {
 		logger.V(logs.LogInfo).Info("cluster is paused. Do nothing.")
+		clusterSummaryScope.ClusterSummary.Status.ReconciliationSuspended = true
+		suspensionReason := clusterPausedMessage
+		clusterSummaryScope.ClusterSummary.Status.SuspensionReason = &suspensionReason
 		return reconcile.Result{}, nil
 	}
 
+	clusterSummaryScope.ClusterSummary.Status.ReconciliationSuspended = false
+	clusterSummaryScope.ClusterSummary.Status.SuspensionReason = nil
+
 	err = r.startWatcherForTemplateResourceRefs(ctx, clusterSummaryScope.ClusterSummary)
 	if err != nil {
 		logger.V(logs.LogInfo).Error(err, "failed to start watcher on resources referenced in TemplateResourceRefs.")
@@ -1588,3 +1587,41 @@ func (r *ClusterSummaryReconciler) updateStatusWithMissingLicenseError(
 	r.updateFeatureStatus(clusterSummaryScope, libsveltosv1beta1.FeatureResources, &failed, nil,
 		notEligibleError, logger)
 }
+
+// removeFinalizer removes the finalizer
+func (r *ClusterSummaryReconciler) removeFinalizer(clusterSummaryScope *scope.ClusterSummaryScope,
+	logger logr.Logger) error {
+
+	// Remove Finalizer
+	logger.V(logs.LogInfo).Info("Removing finalizer")
+	if controllerutil.ContainsFinalizer(clusterSummaryScope.ClusterSummary, configv1beta1.ClusterSummaryFinalizer) {
+		if finalizersUpdated := controllerutil.RemoveFinalizer(clusterSummaryScope.ClusterSummary,
+			configv1beta1.ClusterSummaryFinalizer); !finalizersUpdated {
+			return fmt.Errorf("failed to remove finalizer")
+		}
+	}
+
+	return nil
+}
+
+func (r *ClusterSummaryReconciler) processUndeployError(clusterSummaryScope *scope.ClusterSummaryScope,
+	undeployError error, logger logr.Logger) (reconcile.Result, error) {
+
+	// In DryRun mode it is expected to always get an error back
+	if !clusterSummaryScope.IsDryRunSync() {
+		logger.V(logs.LogInfo).Error(undeployError, "failed to undeploy")
+		return reconcile.Result{Requeue: true, RequeueAfter: deleteRequeueAfter}, nil
+	}
+
+	var nonRetriableError *configv1beta1.NonRetriableError
+	if errors.As(undeployError, &nonRetriableError) {
+		return reconcile.Result{Requeue: true, RequeueAfter: deleteHandOverRequeueAfter}, nil
+	}
+
+	var templateError *configv1beta1.TemplateInstantiationError
+	if errors.As(undeployError, &templateError) {
+		return reconcile.Result{Requeue: true, RequeueAfter: deleteHandOverRequeueAfter}, nil
+	}
+
+	return reconcile.Result{Requeue: true, RequeueAfter: deleteRequeueAfter}, nil
+}

controllers/profile_controller.go

@@ -146,9 +146,14 @@ func (r *ProfileReconciler) Reconcile(ctx context.Context, req ctrl.Request) (_
 
 	if isProfilePaused(profileScope) {
 		logger.V(logs.LogInfo).Info("profile is paused. Skip reconciliation")
+		profileScope.GetStatus().ReconciliationSuspended = true
+		suspensionReason := "Profile is paused"
+		profileScope.GetStatus().SuspensionReason = &suspensionReason
 		return reconcile.Result{}, nil
 	}
 
+	profileScope.GetStatus().ReconciliationSuspended = false
+	profileScope.GetStatus().SuspensionReason = nil
 	// Always close the scope when exiting this function so we can persist any Profile
 	// changes.
 	defer func() {

manifest/manifest.yaml

@@ -1539,6 +1539,16 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              reconciliationSuspended:
+                description: |-
+                  ReconciliationSuspended indicates whether the reconciliation loop for this
+                  ClusterSummary is currently paused due to an external action (e.g., a user annotation).
+                  When true, the status will not be updated unless the pause is lifted.
+                type: boolean
+              suspensionReason:
+                description: SuspensionReason provides a brief explanation of why
+                  the reconciliation is suspended.
+                type: string
               updatedClusters:
                 description: |-
                   UpdatedClusters contains information all the cluster currently matching
@@ -3354,6 +3364,16 @@ spec:
                   If not set, reconciliations will happen as usual.
                 format: date-time
                 type: string
+              reconciliationSuspended:
+                description: |-
+                  ReconciliationSuspended indicates whether the reconciliation loop for this
+                  ClusterSummary is currently paused due to an external action (e.g., a user annotation).
+                  When true, the status will not be updated unless the pause is lifted.
+                type: boolean
+              suspensionReason:
+                description: SuspensionReason provides a brief explanation of why
+                  the reconciliation is suspended.
+                type: string
             type: object
         type: object
     served: true
@@ -4592,6 +4612,16 @@ spec:
                   type: object
                   x-kubernetes-map-type: atomic
                 type: array
+              reconciliationSuspended:
+                description: |-
+                  ReconciliationSuspended indicates whether the reconciliation loop for this
+                  ClusterSummary is currently paused due to an external action (e.g., a user annotation).
+                  When true, the status will not be updated unless the pause is lifted.
+                type: boolean
+              suspensionReason:
+                description: SuspensionReason provides a brief explanation of why
+                  the reconciliation is suspended.
+                type: string
               updatedClusters:
                 description: |-
                   UpdatedClusters contains information all the cluster currently matching

test/fv/dryrun_test.go

@@ -738,11 +738,6 @@ func verifyResourceReport(clusterReport *configv1beta1.ClusterReport,
 }
 
 func verifyDeployedGroupVersionKind(clusterProfileName string) {
-	Byf("Verifying DeployedGroupVersionKind are set for ClusterProfile %s", clusterProfileName)
-	// Test has been flaky. Rarely it happens that Kong service is not removed
-	// when clusterProfile is.
-	// Adding this extra code to make test fails if at this points, ClusterSummary
-	// has lost list of deployed GVKs (which will cause the cleanup to not happen)
 	listOptions := []client.ListOption{
 		client.MatchingLabels{
 			clusterops.ClusterProfileLabelName: clusterProfileName,
@@ -751,15 +746,32 @@ func verifyDeployedGroupVersionKind(clusterProfileName string) {
 	clusterSummaryList := &configv1beta1.ClusterSummaryList{}
 	Expect(k8sClient.List(context.TODO(), clusterSummaryList, listOptions...)).To(Succeed())
 	Expect(len(clusterSummaryList.Items)).To(Equal(1))
-	found := false
-	for i := range clusterSummaryList.Items[0].Status.DeployedGVKs {
-		fs := clusterSummaryList.Items[0].Status.DeployedGVKs[i]
-		if fs.FeatureID == libsveltosv1beta1.FeatureResources {
-			Expect(len(fs.DeployedGroupVersionKind)).ToNot(BeZero())
-			found = true
+
+	Byf("Verifying DeployedGroupVersionKind are set for ClusterSummary %s/%s",
+		clusterSummaryList.Items[0].Namespace, clusterSummaryList.Items[0].Name)
+	// Test has been flaky. Rarely it happens that Kong service is not removed
+	// when clusterProfile is.
+	// Adding this extra code to make test fails if at this points, ClusterSummary
+	// has lost list of deployed GVKs (which will cause the cleanup to not happen)
+
+	Eventually(func() bool {
+		currentClusterSummary := &configv1beta1.ClusterSummary{}
+		err := k8sClient.Get(context.TODO(),
+			types.NamespacedName{
+				Namespace: clusterSummaryList.Items[0].Namespace,
+				Name:      clusterSummaryList.Items[0].Name},
+			currentClusterSummary)
+		if err != nil {
+			return false
 		}
-	}
-	Expect(found).To(BeTrue())
+		for i := range currentClusterSummary.Status.DeployedGVKs {
+			fs := currentClusterSummary.Status.DeployedGVKs[i]
+			if fs.FeatureID == libsveltosv1beta1.FeatureResources {
+				return len(fs.DeployedGroupVersionKind) != 0
+			}
+		}
+		return false
+	}, timeout, pollingInterval).Should(BeTrue())
 }
 
 func setTier(clusterProfileName string, tier int32) {