CHAOSPLT-580: Send metric when cron or rollout self-deletes (#953)

* CHAOSPLT-580: Allow for alerting when a cron is deleted for missing its target

* allow for configuring target resource missing threshold

Author: ptnapoleon

chart/templates/configmap.yaml

@@ -65,6 +65,7 @@ data:
       minimumCronFrequency: {{ .Values.controller.minimumCronFrequency }}
       maxDuration: {{ .Values.controller.maxDuration }}
       finalizerDeletionDelay: {{ .Values.controller.finalizerDeletionDelay }}
+      targetResourceMissingThreshold: {{ .Values.controller.targetResourceMissingThreshold }}
       expiredDisruptionGCDelay: {{ .Values.controller.expiredDisruptionGCDelay }}
       userInfoHook: {{ .Values.controller.userInfoHook }}
       webhook:

chart/values.yaml

@@ -71,6 +71,7 @@ controller:
   defaultCronDelayedStartTolerance: 15m
   minimumCronFrequency: 15m # a disruption cron with a spec.schedule that runs more often than this will be rejected.
   finalizerDeletionDelay: 20s
+  targetResourceMissingThreshold: 24h # duration after a cron or rollout self-delete if target is missing for this long
   expiredDisruptionGCDelay: 10m # time after a disruption expires before deleting it
   userInfoHook: true
   webhook: # admission webhook configuration

chart/values/local.yaml

@@ -43,6 +43,7 @@ controller:
   defaultDuration: 3m
   finalizerDeletionDelay: 2s
   expiredDisruptionGCDelay: 15s
+  targetResourceMissingThreshold: 1m
   webhook:
     certDir: chart/certs
     host: ""

config/config.go

@@ -55,6 +55,7 @@ type controllerConfig struct {
 	DisruptionRolloutEnabled         bool                            `json:"disruptionRolloutEnabled" yaml:"disruptionRolloutEnabled"`
 	DisruptionDeletionTimeout        time.Duration                   `json:"disruptionDeletionTimeout" yaml:"disruptionDeletionTimeout"`
 	FinalizerDeletionDelay           time.Duration                   `json:"finalizerDeletionDelay" yaml:"finalizerDeletionDelay"`
+	TargetResourceMissingThreshold   time.Duration                   `json:"targetResourceMissingThreshold" yaml:"targetResourceMissingThreshold"`
 	DisabledDisruptions              []string                        `json:"disabledDisruptions" yaml:"disabledDisruptions"`
 }
 
@@ -569,6 +570,12 @@ func New(client corev1client.ConfigMapInterface, logger *zap.SugaredLogger, osAr
 		return cfg, err
 	}
 
+	mainFS.DurationVar(&cfg.Controller.TargetResourceMissingThreshold, "target-resource-missing-threshold", time.Hour*24, "Define the amount of time a cron or rollout will tolerate its target missing before self-deleting")
+
+	if err := viper.BindPFlag("controller.targetResourceMissingThreshold", mainFS.Lookup("target-resource-missing-threshold")); err != nil {
+		return cfg, err
+	}
+
 	if err := preConfigFS.Parse(osArgs); err != nil {
 		return cfg, fmt.Errorf("unable to retrieve configuration parse from provided flag: %w", err)
 	}

controllers/cron_rollout_helpers.go

@@ -25,9 +25,8 @@ import (
 )
 
 const (
-	DisruptionCronNameLabel        = chaosv1beta1.GroupName + "/disruption-cron-name"
-	DisruptionRolloutNameLabel     = chaosv1beta1.GroupName + "/disruption-rollout-name"
-	TargetResourceMissingThreshold = time.Hour * 24
+	DisruptionCronNameLabel    = chaosv1beta1.GroupName + "/disruption-cron-name"
+	DisruptionRolloutNameLabel = chaosv1beta1.GroupName + "/disruption-rollout-name"
 )
 
 // GetChildDisruptions retrieves disruptions associated with a resource by its label.

controllers/disruption_cron_controller.go

@@ -27,12 +27,13 @@ import (
 var DisruptionCronTags = []string{}
 
 type DisruptionCronReconciler struct {
-	Client                 client.Client
-	Scheme                 *runtime.Scheme
-	BaseLog                *zap.SugaredLogger
-	log                    *zap.SugaredLogger
-	MetricsSink            metrics.Sink
-	FinalizerDeletionDelay time.Duration
+	Client                         client.Client
+	Scheme                         *runtime.Scheme
+	BaseLog                        *zap.SugaredLogger
+	log                            *zap.SugaredLogger
+	MetricsSink                    metrics.Sink
+	FinalizerDeletionDelay         time.Duration
+	TargetResourceMissingThreshold time.Duration
 }
 
 func (r *DisruptionCronReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
@@ -260,7 +261,7 @@ func (r *DisruptionCronReconciler) updateTargetResourcePreviouslyMissing(ctx con
 			return targetResourceExists, disruptionCronDeleted, r.handleTargetResourceFirstMissing(ctx, instance)
 		}
 
-		if time.Since(instance.Status.TargetResourcePreviouslyMissing.Time) > TargetResourceMissingThreshold {
+		if time.Since(instance.Status.TargetResourcePreviouslyMissing.Time) > r.TargetResourceMissingThreshold {
 			r.log.Warnw("target has been missing for over one day, deleting this schedule",
 				"timeMissing", time.Since(instance.Status.TargetResourcePreviouslyMissing.Time))
 
@@ -298,6 +299,8 @@ func (r *DisruptionCronReconciler) handleTargetResourceMissingPastExpiration(ctx
 		return fmt.Errorf("failed to delete instance: %w", err)
 	}
 
+	r.handleMetricSinkError(r.MetricsSink.MetricMissingTargetDeleted(DisruptionCronTags))
+
 	return nil
 }
 

controllers/disruption_rollout_controller.go

@@ -24,11 +24,12 @@ import (
 var DisruptionRolloutTags = []string{}
 
 type DisruptionRolloutReconciler struct {
-	Client      client.Client
-	Scheme      *runtime.Scheme
-	BaseLog     *zap.SugaredLogger
-	log         *zap.SugaredLogger
-	MetricsSink metrics.Sink
+	Client                         client.Client
+	Scheme                         *runtime.Scheme
+	BaseLog                        *zap.SugaredLogger
+	log                            *zap.SugaredLogger
+	MetricsSink                    metrics.Sink
+	TargetResourceMissingThreshold time.Duration
 }
 
 func (r *DisruptionRolloutReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, err error) {
@@ -203,7 +204,7 @@ func (r *DisruptionRolloutReconciler) updateTargetResourcePreviouslyMissing(ctx
 			return targetResourceExists, disruptionRolloutDeleted, r.handleTargetResourceFirstMissing(ctx, instance)
 		}
 
-		if time.Since(instance.Status.TargetResourcePreviouslyMissing.Time) > TargetResourceMissingThreshold {
+		if time.Since(instance.Status.TargetResourcePreviouslyMissing.Time) > r.TargetResourceMissingThreshold {
 			r.log.Errorw("target has been missing for over one day, deleting this schedule",
 				"timeMissing", time.Since(instance.Status.TargetResourcePreviouslyMissing.Time))
 
@@ -241,6 +242,8 @@ func (r *DisruptionRolloutReconciler) handleTargetResourceMissingPastExpiration(
 		return fmt.Errorf("failed to delete instance: %w", err)
 	}
 
+	r.handleMetricSinkError(r.MetricsSink.MetricMissingTargetDeleted(DisruptionRolloutTags))
+
 	return nil
 }
 

main.go

@@ -322,7 +322,8 @@ func main() {
 			BaseLog: logger,
 			Scheme:  mgr.GetScheme(),
 			// new metrics sink for rollout controller
-			MetricsSink: initMetricsSink(cfg.Controller.MetricsSink, logger, metricstypes.SinkAppRolloutController),
+			MetricsSink:                    initMetricsSink(cfg.Controller.MetricsSink, logger, metricstypes.SinkAppRolloutController),
+			TargetResourceMissingThreshold: cfg.Controller.TargetResourceMissingThreshold,
 		}
 
 		defer closeMetricsSink(logger, disruptionRolloutReconciler.MetricsSink)
@@ -386,8 +387,9 @@ func main() {
 			BaseLog: logger,
 			Scheme:  mgr.GetScheme(),
 			// new metrics sink for cron controller
-			MetricsSink:            initMetricsSink(cfg.Controller.MetricsSink, logger, metricstypes.SinkAppCronController),
-			FinalizerDeletionDelay: cfg.Controller.FinalizerDeletionDelay,
+			MetricsSink:                    initMetricsSink(cfg.Controller.MetricsSink, logger, metricstypes.SinkAppCronController),
+			FinalizerDeletionDelay:         cfg.Controller.FinalizerDeletionDelay,
+			TargetResourceMissingThreshold: cfg.Controller.TargetResourceMissingThreshold,
 		}
 
 		defer closeMetricsSink(logger, disruptionCronReconciler.MetricsSink)

o11y/metrics/datadog/datadog.go

@@ -242,6 +242,12 @@ func (d Sink) MetricMissingTargetFound(tags []string) error {
 	return d.metricWithStatus(d.prefix+"schedule.missing_target_found", tags)
 }
 
+// MetricMissingTargetDeleted reports when a scheduled Disruption has been deleted by the chaos-controller,
+// because its target has been missing for too long
+func (d Sink) MetricMissingTargetDeleted(tags []string) error {
+	return d.metricWithStatus(d.prefix+"schedule.missing_target_deleted", tags)
+}
+
 // MetricNextScheduledTime reports the duration until the next scheduled disruption will run
 func (d Sink) MetricNextScheduledTime(duration time.Duration, tags []string) error {
 	return d.timing(d.prefix+"schedule.next_scheduled", duration, tags)

o11y/metrics/metrics.go

@@ -49,6 +49,7 @@ type Sink interface {
 	MetricTooLate(tags []string) error
 	MetricTargetMissing(duration time.Duration, tags []string) error
 	MetricMissingTargetFound(tags []string) error
+	MetricMissingTargetDeleted(tags []string) error
 	MetricNextScheduledTime(time time.Duration, tags []string) error
 	MetricDisruptionScheduled(tags []string) error
 	MetricPausedCron(tags []string) error

o11y/metrics/noop/noop.go

@@ -234,6 +234,14 @@ func (n Sink) MetricMissingTargetFound(tags []string) error {
 	return nil
 }
 
+// MetricMissingTargetDeleted reports when a scheduled Disruption has been deleted by the chaos-controller,
+// because its target has been missing for too long
+func (n Sink) MetricMissingTargetDeleted(tags []string) error {
+	n.log.Debugf("NOOP: MetricMissingTargetDeleted %s\n", tags)
+
+	return nil
+}
+
 // MetricNextScheduledTime reports the duration until the next scheduled disruption will run
 func (n Sink) MetricNextScheduledTime(duration time.Duration, tags []string) error {
 	n.log.Debugf("NOOP: MetricNextScheduledRun %v, s%s\n", duration, tags)