chore: Add operator_status_condition_current_status_seconds and operator_status_condition_transitions_total (#62)

* Add status condition metrics on last_transition_time and current_status_seconds

* Drop last_condition_time testing

* Add testing for metrics on abnormal conditions

Author: jonathan-innis

status/controller.go

@@ -3,6 +3,7 @@ package status
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/awslabs/operatorpkg/object"
 	"github.com/prometheus/client_golang/prometheus"
@@ -12,6 +13,7 @@ import (
 	"k8s.io/client-go/tools/record"
 	controllerruntime "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/metrics"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
@@ -24,6 +26,7 @@ const (
 	MetricLabelName            = "name"
 	MetricLabelConditionType   = "type"
 	MetricLabelConditionStatus = "status"
+	MetricLabelConditionReason = "reason"
 )
 
 const (
@@ -45,9 +48,10 @@ func NewController[T Object](client client.Client, eventRecorder record.EventRec
 	}
 }
 
-func (c *Controller[T]) Register(ctx context.Context, m manager.Manager) error {
+func (c *Controller[T]) Register(_ context.Context, m manager.Manager) error {
 	return controllerruntime.NewControllerManagedBy(m).
 		For(object.New[T]()).
+		WithOptions(controller.Options{MaxConcurrentReconciles: 10}).
 		Named("status").
 		Complete(c)
 }
@@ -61,8 +65,14 @@ func (c *Controller[T]) Reconcile(ctx context.Context, req reconcile.Request) (r
 			ConditionCount.DeletePartialMatch(prometheus.Labels{
 				MetricLabelGroup:     gvk.Group,
 				MetricLabelKind:      gvk.Kind,
-				MetricLabelNamespace: string(req.Namespace),
-				MetricLabelName:      string(req.Name),
+				MetricLabelNamespace: req.Namespace,
+				MetricLabelName:      req.Name,
+			})
+			ConditionCurrentStatusSeconds.DeletePartialMatch(prometheus.Labels{
+				MetricLabelGroup:     gvk.Group,
+				MetricLabelKind:      gvk.Kind,
+				MetricLabelNamespace: req.Namespace,
+				MetricLabelName:      req.Name,
 			})
 			return reconcile.Result{}, nil
 		}
@@ -78,21 +88,41 @@ func (c *Controller[T]) Reconcile(ctx context.Context, req reconcile.Request) (r
 		ConditionCount.With(prometheus.Labels{
 			MetricLabelGroup:           gvk.Group,
 			MetricLabelKind:            gvk.Kind,
-			MetricLabelNamespace:       string(req.Namespace),
-			MetricLabelName:            string(req.Name),
-			MetricLabelConditionType:   string(condition.Type),
+			MetricLabelNamespace:       req.Namespace,
+			MetricLabelName:            req.Name,
+			MetricLabelConditionType:   condition.Type,
 			MetricLabelConditionStatus: string(condition.Status),
+			MetricLabelConditionReason: condition.Reason,
 		}).Set(1)
+		ConditionCurrentStatusSeconds.With(prometheus.Labels{
+			MetricLabelGroup:           gvk.Group,
+			MetricLabelKind:            gvk.Kind,
+			MetricLabelNamespace:       req.Namespace,
+			MetricLabelName:            req.Name,
+			MetricLabelConditionType:   condition.Type,
+			MetricLabelConditionStatus: string(condition.Status),
+			MetricLabelConditionReason: condition.Reason,
+		}).Set(time.Since(condition.LastTransitionTime.Time).Seconds())
 	}
 	for _, observedCondition := range observedConditions.List() {
 		if currentCondition := currentConditions.Get(observedCondition.Type); currentCondition == nil || currentCondition.Status != observedCondition.Status {
 			ConditionCount.Delete(prometheus.Labels{
 				MetricLabelGroup:           gvk.Group,
 				MetricLabelKind:            gvk.Kind,
-				MetricLabelNamespace:       string(req.Namespace),
-				MetricLabelName:            string(req.Name),
-				MetricLabelConditionType:   string(observedCondition.Type),
+				MetricLabelNamespace:       req.Namespace,
+				MetricLabelName:            req.Name,
+				MetricLabelConditionType:   observedCondition.Type,
 				MetricLabelConditionStatus: string(observedCondition.Status),
+				MetricLabelConditionReason: observedCondition.Reason,
+			})
+			ConditionCurrentStatusSeconds.Delete(prometheus.Labels{
+				MetricLabelGroup:           gvk.Group,
+				MetricLabelKind:            gvk.Kind,
+				MetricLabelNamespace:       req.Namespace,
+				MetricLabelName:            req.Name,
+				MetricLabelConditionType:   observedCondition.Type,
+				MetricLabelConditionStatus: string(observedCondition.Status),
+				MetricLabelConditionReason: observedCondition.Reason,
 			})
 		}
 	}
@@ -114,25 +144,36 @@ func (c *Controller[T]) Reconcile(ctx context.Context, req reconcile.Request) (r
 	// time, and our likelyhood of observing this is much higher.
 	for _, condition := range currentConditions.List() {
 		observedCondition := observedConditions.Get(condition.Type)
-		if observedCondition == nil || observedCondition.GetStatus() == condition.GetStatus() {
+		if observedCondition.GetStatus() == condition.GetStatus() {
+			continue
+		}
+		// A condition transitions if it either didn't exist before or it has changed
+		ConditionTransitionsTotal.With(prometheus.Labels{
+			MetricLabelGroup:           gvk.Group,
+			MetricLabelKind:            gvk.Kind,
+			MetricLabelConditionType:   condition.Type,
+			MetricLabelConditionStatus: string(condition.Status),
+			MetricLabelConditionReason: condition.Reason,
+		}).Inc()
+		if observedCondition == nil {
 			continue
 		}
 		duration := condition.LastTransitionTime.Time.Sub(observedCondition.LastTransitionTime.Time).Seconds()
 		ConditionDuration.With(prometheus.Labels{
 			MetricLabelGroup:           gvk.Group,
 			MetricLabelKind:            gvk.Kind,
-			MetricLabelConditionType:   string(observedCondition.Type),
+			MetricLabelConditionType:   observedCondition.Type,
 			MetricLabelConditionStatus: string(observedCondition.Status),
-		}).Observe(float64(duration))
-		c.eventRecorder.Event(o, v1.EventTypeNormal, string(condition.Type), fmt.Sprintf("Status condition transitioned, Type: %s, Status: %s -> %s, Reason: %s%s",
+		}).Observe(duration)
+		c.eventRecorder.Event(o, v1.EventTypeNormal, condition.Type, fmt.Sprintf("Status condition transitioned, Type: %s, Status: %s -> %s, Reason: %s%s",
 			condition.Type,
 			observedCondition.Status,
 			condition.Status,
 			condition.Reason,
 			lo.Ternary(condition.Message != "", fmt.Sprintf(", Message: %s", condition.Message), ""),
 		))
 	}
-	return reconcile.Result{}, nil
+	return reconcile.Result{RequeueAfter: time.Second * 10}, nil
 }
 
 // Cardinality is limited to # objects * # conditions * # objectives
@@ -166,12 +207,53 @@ var ConditionCount = prometheus.NewGaugeVec(
 		MetricLabelKind,
 		MetricLabelConditionType,
 		MetricLabelConditionStatus,
+		MetricLabelConditionReason,
+	},
+)
+
+// Cardinality is limited to # objects * # conditions
+// NOTE: This metric is based on a requeue so it won't show the current status seconds with extremely high accuracy.
+// This metric is useful for aggreations. If you need a high accuracy metric, use operator_status_condition_last_transition_time_seconds
+var ConditionCurrentStatusSeconds = prometheus.NewGaugeVec(
+	prometheus.GaugeOpts{
+		Namespace: MetricNamespace,
+		Subsystem: MetricSubsystem,
+		Name:      "current_status_seconds",
+		Help:      "The current amount of time in seconds that a status condition has been in a specific state. Alarm := P99(Updated=Unknown) > 5 minutes",
+	},
+	[]string{
+		MetricLabelNamespace,
+		MetricLabelName,
+		MetricLabelGroup,
+		MetricLabelKind,
+		MetricLabelConditionType,
+		MetricLabelConditionStatus,
+		MetricLabelConditionReason,
+	},
+)
+
+// Cardinality is limited to # objects * # conditions
+var ConditionTransitionsTotal = prometheus.NewCounterVec(
+	prometheus.CounterOpts{
+		Namespace: MetricNamespace,
+		Subsystem: MetricSubsystem,
+		Name:      "transitions_total",
+		Help:      "The count of transitions of a given object, type and status.",
+	},
+	[]string{
+		MetricLabelGroup,
+		MetricLabelKind,
+		MetricLabelConditionType,
+		MetricLabelConditionStatus,
+		MetricLabelConditionReason,
 	},
 )
 
 func init() {
 	metrics.Registry.MustRegister(
 		ConditionCount,
 		ConditionDuration,
+		ConditionTransitionsTotal,
+		ConditionCurrentStatusSeconds,
 	)
 }