KUBESAW-264: Implement TierTemplateRevision cleanup controller

controllers/nstemplatetierrevisioncleanup/nstemplatetier_revision_cleanup_controller.go

@@ -0,0 +1,133 @@
+package nstemplatetierrevisioncleanup
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"sigs.k8s.io/controller-runtime/pkg/log"
+
+	toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1"
+	"github.com/codeready-toolchain/host-operator/controllers/nstemplatetier"
+	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	ctrl "sigs.k8s.io/controller-runtime"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+)
+
+const deletionTimeThreshold = 30 * time.Second
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *Reconciler) SetupWithManager(mgr manager.Manager) error {
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&toolchainv1alpha1.TierTemplateRevision{}).
+		Complete(r)
+}
+
+type Reconciler struct {
+	Client runtimeclient.Client
+}
+
+func (r *Reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+
+	// fetch the NSTemplateTier tier
+	ttr := &toolchainv1alpha1.TierTemplateRevision{}
+	if err := r.Client.Get(ctx, request.NamespacedName, ttr); err != nil {
+		if errors.IsNotFound(err) {
+			logger.Info("TierTemplateRevision not found")
+			return reconcile.Result{}, nil
+		}
+		// Error reading the object - requeue the request.
+		logger.Error(err, "unable to get the current TierTemplateRevision")
+		return reconcile.Result{}, fmt.Errorf("unable to get the current TierTemplateRevision: %w", err)
+	}
+	//there is no point in fetching  the NStemplateTier, if the TTR  is just created
+
+	//get the tier template revision creation time stamp and the duration
+	timeSinceCreation := time.Since(ttr.GetCreationTimestamp().Time)
+
+	//the ttr age should be greater than 30 seconds
+	if timeSinceCreation < deletionTimeThreshold {
+		requeAfter := deletionTimeThreshold - timeSinceCreation
+		return reconcile.Result{RequeueAfter: requeAfter, Requeue: true}, nil
+	}
+
+	//check if there is tier-name label available
+	tierName, ok := ttr.GetLabels()[toolchainv1alpha1.TierLabelKey]
+	if !ok {
+		logger.Info("tier-name label not found in tiertemplaterevision")
+		return reconcile.Result{}, nil
+
+	}
+	// fetch the related NSTemplateTier tier
+	tier := &toolchainv1alpha1.NSTemplateTier{}
+	if err := r.Client.Get(ctx, types.NamespacedName{
+		Namespace: ttr.GetNamespace(),
+		Name:      tierName,
+	}, tier); err != nil {
+		if errors.IsNotFound(err) {
+			logger.Info("NSTemplateTier not found")
+		}
+		// Error reading the object - requeue the request.
+		logger.Error(err, "unable to get the current NSTemplateTier")
+		return reconcile.Result{}, fmt.Errorf("unable to get the current NSTemplateTier: %w", err)
+	}
+
+	// verify that the tier template revision which is unsused(not referenced by any tiers and whose creation date is older that 30secs
+	// and all the spaces are up-to-date)
+	unusedRevisionBool, err := r.VerifyUnusedTTR(ctx, tier, ttr)
+	if err != nil {
+		return reconcile.Result{}, err
+	}
+
+	// Delete the unused revision
+	if unusedRevisionBool {
+		if err := r.Client.Delete(ctx, ttr); err != nil {
+			return reconcile.Result{}, fmt.Errorf("unable to delete the current Tier Template Revision %s: %w", ttr.Name, err)
+		}
+	}
+
+	return reconcile.Result{}, nil
+}
+
+// VerifyUnusedTTR function verifies that the tier template revision's creation time stamp is older than 30sec,
+// it is not present/refernced in status.revisions field, and there are no outdated spaces.
+func (r *Reconciler) VerifyUnusedTTR(ctx context.Context, nsTmplTier *toolchainv1alpha1.NSTemplateTier,
+	rev *toolchainv1alpha1.TierTemplateRevision) (bool, error) {
+
+	logger := log.FromContext(ctx)
+	//check if the ttr name is present status.revisions
+	for _, ttStatusRev := range nsTmplTier.Status.Revisions {
+		if ttStatusRev == rev.Name {
+			logger.Info("the revision is still being referenced in status.revisions")
+			return false, nil
+		}
+	}
+
+	// get the outdated matchig label to list outdated spaces
+	matchOutdated, err := nstemplatetier.OutdatedTierSelector(nsTmplTier)
+	if err != nil {
+		return false, err
+
+	}
+	// look-up all spaces associated with the NSTemplateTier which are outdated
+	spaces := &toolchainv1alpha1.SpaceList{}
+	if err := r.Client.List(ctx, spaces, runtimeclient.InNamespace(nsTmplTier.Namespace),
+		matchOutdated, runtimeclient.Limit(1)); err != nil {
+		return false, err
+	}
+
+	//If there has been an update on nstemplatetier, it might be in a process to update all the spaces.
+	// so we need to check that that there should not be any outdated spaces.
+	if len(spaces.Items) > 0 {
+		logger.Info("there are still some spaces which are outdated")
+		return false, nil
+	}
+
+	return true, nil
+
+}

controllers/nstemplatetierrevisioncleanup/nstemplatetier_revision_cleanup_controller_test.go

@@ -0,0 +1,217 @@
+package nstemplatetierrevisioncleanup_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	toolchainv1alpha1 "github.com/codeready-toolchain/api/api/v1alpha1"
+	"github.com/codeready-toolchain/host-operator/controllers/nstemplatetierrevisioncleanup"
+	"github.com/codeready-toolchain/host-operator/pkg/apis"
+	tiertest "github.com/codeready-toolchain/host-operator/test/nstemplatetier"
+	"github.com/codeready-toolchain/host-operator/test/tiertemplaterevision"
+	"github.com/codeready-toolchain/toolchain-common/pkg/test"
+	spacetest "github.com/codeready-toolchain/toolchain-common/pkg/test/space"
+	"github.com/stretchr/testify/require"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/kubectl/pkg/scheme"
+	controllerruntime "sigs.k8s.io/controller-runtime"
+	runtimeclient "sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+)
+
+const (
+	operatorNamespace = "toolchain-host-operator"
+)
+
+func TestTTRDeletionReconcile(t *testing.T) {
+	t.Run("TTR Deleted Successfully", func(t *testing.T) {
+		//given
+		nsTemplateTier := tiertest.Base1nsTier(t, tiertest.CurrentBase1nsTemplates, tiertest.WithStatusRevisions())
+		ttr := createttr(*nsTemplateTier, (nsTemplateTier.Spec.ClusterResources.TemplateRef + "-ttrcr"), metav1.NewTime(time.Now().Add(-time.Minute)))
+		s := createSpace(nsTemplateTier)
+		r, req, cl := prepareReconcile(t, ttr.Name, ttr, s, nsTemplateTier)
+		//when
+		res, err := r.Reconcile(context.TODO(), req)
+		//then
+		require.NoError(t, err)
+		require.Equal(t, controllerruntime.Result{}, res)
+		tiertemplaterevision.AssertThatTTRs(t, cl, nsTemplateTier.GetNamespace()).DoNotExist()
+	})
+	t.Run("Failure", func(t *testing.T) {
+		nsTemplateTier := tiertest.Base1nsTier(t, tiertest.CurrentBase1nsTemplates, tiertest.WithStatusRevisions())
+		ttr := createttr(*nsTemplateTier, (nsTemplateTier.Spec.ClusterResources.TemplateRef + "-ttrcr"), metav1.NewTime(time.Now().Add(-time.Minute)))
+		s := createSpace(nsTemplateTier)
+		t.Run("the creation timestamp is less than 30 sec", func(t *testing.T) {
+			// given
+			ttr := createttr(*nsTemplateTier, (nsTemplateTier.Spec.ClusterResources.TemplateRef + "-ttr"), metav1.NewTime(time.Now()))
+			r, req, cl := prepareReconcile(t, ttr.Name, ttr, s, nsTemplateTier)
+
+			// when
+			res, err := r.Reconcile(context.TODO(), req)
+
+			// then
+			require.NoError(t, err)
+			require.True(t, res.Requeue)
+			tiertemplaterevision.AssertThatTTRs(t, cl, nsTemplateTier.GetNamespace()).ExistFor(nsTemplateTier.Name)
+
+		})
+		t.Run("ttr is still being referenced in status.revisions", func(t *testing.T) {
+			// given
+			ttr := createttr(*nsTemplateTier, (nsTemplateTier.Spec.ClusterResources.TemplateRef + "-ttr"), metav1.NewTime(time.Now().Add(-time.Minute)))
+			r, req, cl := prepareReconcile(t, ttr.Name, ttr, s, nsTemplateTier)
+
+			// when
+			res, err := r.Reconcile(context.TODO(), req)
+
+			// then
+			require.NoError(t, err)
+			require.False(t, res.Requeue)
+			tiertemplaterevision.AssertThatTTRs(t, cl, nsTemplateTier.GetNamespace()).ExistFor(nsTemplateTier.Name)
+
+		})
+
+		t.Run("spaces are still being updated", func(t *testing.T) {
+			// given
+			nsTemplateTier.Status.Revisions = map[string]string{
+				"base1ns-code-123456new":             "base1ns-code-123456new-ttr",
+				"base1ns-clusterresources-123456new": "base1ns-clusterresources-123456new-ttr",
+			}
+			ttr := createttr(*nsTemplateTier, (nsTemplateTier.Spec.ClusterResources.TemplateRef + "-ttrcr"), metav1.NewTime(time.Now().Add(-time.Minute)))
+			r, req, cl := prepareReconcile(t, ttr.Name, ttr, s, nsTemplateTier)
+
+			// when
+			res, err := r.Reconcile(context.TODO(), req)
+
+			// then
+			require.NoError(t, err)
+			require.False(t, res.Requeue)
+			tiertemplaterevision.AssertThatTTRs(t, cl, nsTemplateTier.GetNamespace()).ExistFor(nsTemplateTier.Name)
+		})
+
+		t.Run("Error while deleting the TTR", func(t *testing.T) {
+			// given
+			nsTemplateTier := tiertest.Base1nsTier(t, tiertest.CurrentBase1nsTemplates, tiertest.WithStatusRevisions())
+			ttr := createttr(*nsTemplateTier, (nsTemplateTier.Spec.ClusterResources.TemplateRef + "-ttrcr"), metav1.NewTime(time.Now().Add(-time.Minute)))
+			s := createSpace(nsTemplateTier)
+			r, req, cl := prepareReconcile(t, ttr.Name, ttr, s, nsTemplateTier)
+			cl.MockDelete = func(ctx context.Context, obj runtimeclient.Object, opts ...runtimeclient.DeleteOption) error {
+				return fmt.Errorf("some error cannot delete")
+			}
+			// when
+			res, err := r.Reconcile(context.TODO(), req)
+
+			// then
+			require.EqualError(t, err, "unable to delete the current Tier Template Revision base1ns-clusterresources-123456new-ttrcr: some error cannot delete")
+			require.False(t, res.Requeue)
+			tiertemplaterevision.AssertThatTTRs(t, cl, nsTemplateTier.GetNamespace()).ExistFor(nsTemplateTier.Name)
+		})
+
+		t.Run("error while getting revision", func(t *testing.T) {
+			r, req, cl := prepareReconcile(t, ttr.Name, ttr, s, nsTemplateTier)
+			cl.MockGet = func(ctx context.Context, key runtimeclient.ObjectKey, obj runtimeclient.Object, opts ...runtimeclient.GetOption) error {
+				return fmt.Errorf("some error cannot get")
+			}
+			// when
+			res, err := r.Reconcile(context.TODO(), req)
+
+			// then
+			require.EqualError(t, err, "unable to get the current TierTemplateRevision: some error cannot get")
+			require.False(t, res.Requeue)
+			tiertemplaterevision.AssertThatTTRs(t, cl, nsTemplateTier.GetNamespace()).ExistFor(nsTemplateTier.Name)
+		})
+
+		t.Run("error while listing outdated spaces", func(t *testing.T) {
+			r, req, cl := prepareReconcile(t, ttr.Name, ttr, s, nsTemplateTier)
+			cl.MockList = func(ctx context.Context, list runtimeclient.ObjectList, opts ...runtimeclient.ListOption) error {
+				return fmt.Errorf("some error")
+			}
+			// when
+			res, err := r.Reconcile(context.TODO(), req)
+
+			// then
+			require.EqualError(t, err, "some error")
+			require.False(t, res.Requeue)
+			tiertemplaterevision.AssertThatTTRs(t, cl, nsTemplateTier.GetNamespace()).ExistFor(nsTemplateTier.Name)
+		})
+
+		t.Run("revision not found", func(t *testing.T) {
+			r, req, _ := prepareReconcile(t, "base")
+			//when
+			res, err := r.Reconcile(context.TODO(), req)
+			//then
+			require.NoError(t, err)
+			require.False(t, res.Requeue)
+
+		})
+
+		t.Run("NsTemplate Tier not found", func(t *testing.T) {
+			r, req, _ := prepareReconcile(t, ttr.Name, ttr)
+			//when
+			res, err := r.Reconcile(context.TODO(), req)
+			//then
+			require.Error(t, err)
+			require.False(t, res.Requeue)
+
+		})
+
+		t.Run("tier label not found", func(t *testing.T) {
+			ttr.Labels = map[string]string{}
+			r, req, _ := prepareReconcile(t, ttr.Name, ttr)
+			//when
+			res, err := r.Reconcile(context.TODO(), req)
+			//then
+			require.NoError(t, err)
+			require.False(t, res.Requeue)
+		})
+
+	})
+
+}
+
+func prepareReconcile(t *testing.T, name string, initObjs ...runtimeclient.Object) (*nstemplatetierrevisioncleanup.Reconciler, reconcile.Request, *test.FakeClient) {
+	os.Setenv("WATCH_NAMESPACE", test.HostOperatorNs)
+	s := scheme.Scheme
+	err := apis.AddToScheme(s)
+	require.NoError(t, err)
+	cl := test.NewFakeClient(t, initObjs...)
+	r := &nstemplatetierrevisioncleanup.Reconciler{
+		Client: cl,
+	}
+	return r, reconcile.Request{
+		NamespacedName: types.NamespacedName{
+			Name:      name,
+			Namespace: operatorNamespace,
+		},
+	}, cl
+}
+func createttr(nsTTier toolchainv1alpha1.NSTemplateTier, name string, crtime metav1.Time) *toolchainv1alpha1.TierTemplateRevision {
+	labels := map[string]string{
+		toolchainv1alpha1.TierLabelKey: nsTTier.Name,
+	}
+	ttr := &toolchainv1alpha1.TierTemplateRevision{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace:         operatorNamespace,
+			Name:              name,
+			Labels:            labels,
+			CreationTimestamp: crtime,
+		},
+		Spec: toolchainv1alpha1.TierTemplateRevisionSpec{
+			Parameters: nsTTier.Spec.Parameters,
+		},
+	}
+	return ttr
+}
+
+func createSpace(nsTTier *toolchainv1alpha1.NSTemplateTier) *toolchainv1alpha1.Space {
+	testSpace := spacetest.NewSpace(test.HostOperatorNs, "oddity1",
+		spacetest.WithTierNameAndHashLabelFor(nsTTier),
+		spacetest.WithSpecTargetCluster("member-1"),
+		spacetest.WithStatusTargetCluster("member-1"),
+		spacetest.WithFinalizer(),
+		spacetest.WithCondition(spacetest.Ready()))
+	return testSpace
+}

main.go

@@ -6,14 +6,16 @@
 	"net/http"
 	"os"
 	goruntime "runtime"
+	"time"
+
 	"sigs.k8s.io/controller-runtime/pkg/cache"
 	metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
-	"time"
 
 	"github.com/codeready-toolchain/host-operator/controllers/deactivation"
 	"github.com/codeready-toolchain/host-operator/controllers/masteruserrecord"
 	"github.com/codeready-toolchain/host-operator/controllers/notification"
 	"github.com/codeready-toolchain/host-operator/controllers/nstemplatetier"
+	"github.com/codeready-toolchain/host-operator/controllers/nstemplatetierrevisioncleanup"
 	"github.com/codeready-toolchain/host-operator/controllers/socialevent"
 	"github.com/codeready-toolchain/host-operator/controllers/space"
 	"github.com/codeready-toolchain/host-operator/controllers/spacebindingcleanup"
@@ -279,6 +281,12 @@
 		setupLog.Error(err, "unable to create controller", "controller", "NSTemplateTier")
 		os.Exit(1)
 	}
+	if err = (&nstemplatetierrevisioncleanup.Reconciler{
+		Client: mgr.GetClient(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "NSTemplatTierRevisionCleanup")
+		os.Exit(1)
+	}
 	if err := (&toolchainconfig.Reconciler{
 		Client:         mgr.GetClient(),
 		GetMembersFunc: commoncluster.GetMemberClusters,