Explain in metrics.go how all metrics are used (#955)

* Add comments to metrics.go to explain when and how all metrics are used

* add to the docs

* fix a namespace tag

Author: ptnapoleon

docs/metrics_events.md

@@ -22,7 +22,15 @@ Here's the list of metrics sent by the controller and the injector.
 * `chaos.controller.disruptions.stuck_on_removal` increments when a disruption is stuck on removal
 * `chaos.controller.disruptions.stuck_on_removal_total` is the total count of existing disruption being flagged as stuck on removal
 * `chaos.controller.disruptions.gauge` is the total count of existing disruption
-* `chaos.controller.disruptions.count` increments when a disruption is created
+* `chaos.controller.disruptions.count` increments when a disruption is finished
+* `chaos.controller.watcher.calls_total` increments each time any watcher handles an OnChange event
+* `chaos.cron.controller.schedule.too_late` increments each time a DisruptionCron has missed the time to schedule its disruption
+* `chaos.cron.controller.schedule.target_missing` increments each time a DisruptionCron cannot find its target
+* `chaos.cron.controller.schedule.missing_target_found` increments each time a DisruptionCron which couldn't find its target, is now able to find it
+* `chaos.cron.controller.schedule.missing_target_deleted` increments each time a DisruptionCron self deletes because its target was missing for too long
+* `chaos.cron.controller.schedule.next_scheduled` is the time between now and when the next disruption for this DisruptionCron should run
+* `chaos.cron.controller.schedule.disruption_scheduled` increments each time a DisruptionCron schedules a child disruption
+* `chaos.cron.controller.schedule.paused` increments each time a DisruptionCron reconciles while in a paused state
 
 #### Admission webhooks
 

o11y/metrics/datadog/datadog.go

@@ -79,7 +79,8 @@ func (d Sink) GetPrefix() string {
 	return d.prefix
 }
 
-// MetricInjected increments the injected metric
+// MetricInjected is used by the chaos-injector to indicate it has finished trying to inject the disruption into the target,
+// the `succeed` bool argument is false if there was an error while injecting.
 func (d Sink) MetricInjected(succeed bool, kind string, tags []string) error {
 	status := boolToStatus(succeed)
 	t := []string{"status:" + status, "kind:" + kind}
@@ -88,7 +89,9 @@ func (d Sink) MetricInjected(succeed bool, kind string, tags []string) error {
 	return d.metricWithStatus(d.prefix+"injected", t)
 }
 
-// MetricReinjected increments the reinjected metric
+// MetricReinjected is used by the chaos-injector to indicate it has finished trying to inject the disruption into the target,
+// the `succeed` bool argument is false if there was an error while injecting. This metric is used instead of MetricInjected
+// if the chaos-injector pod is performing any injection after its first, i.e., when using the pulse feature
 func (d Sink) MetricReinjected(succeed bool, kind string, tags []string) error {
 	status := boolToStatus(succeed)
 	t := []string{"status:" + status, "kind:" + kind}
@@ -97,7 +100,8 @@ func (d Sink) MetricReinjected(succeed bool, kind string, tags []string) error {
 	return d.metricWithStatus(d.prefix+"reinjected", t)
 }
 
-// MetricCleanedForReinjection increments the cleanedForReinjection metric
+// MetricCleanedForReinjection is used by the chaos-injector to indicate an injector has cleaned the disruptions from the target,
+// but expects to reinject, i.e., when using the spec.pulse feature
 func (d Sink) MetricCleanedForReinjection(succeed bool, kind string, tags []string) error {
 	status := boolToStatus(succeed)
 	t := []string{"status:" + status, "kind:" + kind}
@@ -106,7 +110,8 @@ func (d Sink) MetricCleanedForReinjection(succeed bool, kind string, tags []stri
 	return d.metricWithStatus(d.prefix+"cleaned_for_reinjection", t)
 }
 
-// MetricCleaned increments the cleaned metric
+// MetricCleaned is used by the chaos-injector to indicate an injector has cleaned the disruptions from the target,
+// and does not intend to re-inject.
 func (d Sink) MetricCleaned(succeed bool, kind string, tags []string) error {
 	status := boolToStatus(succeed)
 	t := []string{"status:" + status, "kind:" + kind}
@@ -115,55 +120,64 @@ func (d Sink) MetricCleaned(succeed bool, kind string, tags []string) error {
 	return d.metricWithStatus(d.prefix+"cleaned", t)
 }
 
-// MetricReconcile increment reconcile metric
+// MetricReconcile is used to count how many times the controller enters any reconcile loop
 func (d Sink) MetricReconcile() error {
 	return d.metricWithStatus(d.prefix+"reconcile", []string{})
 }
 
-// MetricReconcileDuration send timing metric for reconcile loop
+// MetricReconcileDuration is used at the end of every reconcile loop to indicate the duration that Reconcile() call spent
 func (d Sink) MetricReconcileDuration(duration time.Duration, tags []string) error {
 	return d.timing(d.prefix+"reconcile.duration", duration, tags)
 }
 
-// MetricCleanupDuration send timing metric for cleanup duration
+// MetricCleanupDuration indicates the duration between a Disruption's deletion timestamp, and when the chaos-controller
+// removes its finalizer
 func (d Sink) MetricCleanupDuration(duration time.Duration, tags []string) error {
 	return d.timing(d.prefix+"cleanup.duration", duration, tags)
 }
 
-// MetricInjectDuration send timing metric for inject duration
+// MetricInjectDuration indicates the duration between a Disruption's creation timestamp, and when it reaches a status
+// of Injected, indicating all chaos-injector pods have injected into their targets, and we've reached the expected count
 func (d Sink) MetricInjectDuration(duration time.Duration, tags []string) error {
 	return d.timing(d.prefix+"inject.duration", duration, tags)
 }
 
-// MetricDisruptionCompletedDuration sends timing metric for entire disruption duration
+// MetricDisruptionCompletedDuration indicates the duration between a Disruption's creation timestamp, and when the chaos-controller
+// removes its finalizer
 func (d Sink) MetricDisruptionCompletedDuration(duration time.Duration, tags []string) error {
 	return d.timing(d.prefix+"disruption.completed_duration", duration, tags)
 }
 
-// MetricDisruptionOngoingDuration sends timing metric for disruption duration so far
+// MetricDisruptionOngoingDuration indicates the duration between a Disruption's creation timestamp, and the current time.
+// This is emitted approximately every one minute
 func (d Sink) MetricDisruptionOngoingDuration(duration time.Duration, tags []string) error {
 	return d.timing(d.prefix+"disruption.ongoing_duration", duration, tags)
 }
 
-// MetricPodsCreated increment pods.created metric
+// MetricPodsCreated is used every time the chaos-controller finishes sending a Create request to the k8s api to
+// schedule a new chaos-injector pod. The `succeed` bool argument is false if there was an error returned.
 func (d Sink) MetricPodsCreated(target, instanceName, namespace string, succeed bool) error {
 	status := boolToStatus(succeed)
-	tags := []string{"target:" + target, "disruptionName:" + instanceName, "status:" + status, "namespace:" + namespace}
+	tags := []string{"target:" + target, "disruptionName:" + instanceName, "status:" + status, "disruptionNamespace:" + namespace}
 
 	return d.metricWithStatus(d.prefix+"pods.created", tags)
 }
 
-// MetricStuckOnRemoval increments disruptions.stuck_on_removal metric
+// MetricStuckOnRemoval is emitted once per minute per disruption, if that disruption is "stuck on removal", i.e.,
+// we have attempted to clean and delete the disruption, but that has not worked, and a human needs to intervene.
 func (d Sink) MetricStuckOnRemoval(tags []string) error {
 	return d.metricWithStatus(d.prefix+"disruptions.stuck_on_removal", tags)
 }
 
-// MetricStuckOnRemovalGauge sends disruptions.stuck_on_removal_total metric containing the gauge of stuck disruptions
+// MetricStuckOnRemovalGauge is emitted once per minute counting the total number of disruptions that are
+// "stuck on removal", i.e., we have attempted to clean and delete the disruption, but that has not worked,
+// and a human needs to intervene.
 func (d Sink) MetricStuckOnRemovalGauge(gauge float64) error {
 	return d.client.Gauge(d.prefix+"disruptions.stuck_on_removal_total", gauge, []string{}, 1)
 }
 
-// MetricDisruptionsGauge sends the disruptions.gauge metric counting ongoing disruptions
+// MetricDisruptionsGauge is emitted once per minute counting the total number of ongoing disruptions per namespace,
+// or if we fail to determine the namespaced metrics, simply the total number of disruptions found
 func (d Sink) MetricDisruptionsGauge(gauge float64, tags []string) error {
 	return d.client.Gauge(d.prefix+"disruptions.gauge", gauge, tags, 1)
 }
@@ -174,37 +188,40 @@ func (d Sink) MetricDisruptionsCount(kind chaostypes.DisruptionKindName, tags []
 	return d.metricWithStatus(d.prefix+"disruptions.count", tags)
 }
 
-// MetricPodsGauge sends the pods.gauge metric counting existing chaos pods
+// MetricPodsGauge is emitted once per minute counting the total number of live chaos pods for all ongoing disruptions
 func (d Sink) MetricPodsGauge(gauge float64) error {
 	return d.client.Gauge(d.prefix+"pods.gauge", gauge, []string{}, 1)
 }
 
-// MetricRestart sends an increment of the controller restart metric
+// MetricRestart is emitted once, every time the manager container of the chaos-controller starts up
 func (d Sink) MetricRestart() error {
 	return d.metricWithStatus(d.prefix+"restart", []string{})
 }
 
-// MetricValidationFailed increments the failed validation metric
+// MetricValidationFailed is emitted in ValidateCreate and ValidateUpdate in the disruption_webhook, specifically and
+// only when DisruptionSpec.Validate() returns an error, OR when trying to remove the finalizer from a disruption with
+// chaos pods.
 func (d Sink) MetricValidationFailed(tags []string) error {
 	return d.metricWithStatus(d.prefix+"validation.failed", tags)
 }
 
-// MetricValidationCreated increments the created validation metric
+// MetricValidationCreated is emitted once per created Disruption, in the webhook after validation completes.
 func (d Sink) MetricValidationCreated(tags []string) error {
 	return d.metricWithStatus(d.prefix+"validation.created", tags)
 }
 
-// MetricValidationUpdated increments the updated validation metric
+// MetricValidationUpdated is emitted once per Disruption update, in the webhook after validation completes
 func (d Sink) MetricValidationUpdated(tags []string) error {
 	return d.metricWithStatus(d.prefix+"validation.updated", tags)
 }
 
-// MetricValidationDeleted increments the deleted validation metric
+// MetricValidationDeleted is emitted once per Disruption delete, in the webhook
 func (d Sink) MetricValidationDeleted(tags []string) error {
 	return d.metricWithStatus(d.prefix+"validation.deleted", tags)
 }
 
-// MetricInformed increments when the pod informer receives an event to process before reconciliation
+// MetricInformed is emitted every time the manager container's informer is called to check a pod in the chaos-controller's
+// namespace, to see if that pod is a chaos-injector pod that needs its Disruption reconciled.
 func (d Sink) MetricInformed(tags []string) error {
 	return d.metricWithStatus(d.prefix+"informed", tags)
 }
@@ -214,24 +231,24 @@ func (d Sink) MetricOrphanFound(tags []string) error {
 	return d.metricWithStatus(d.prefix+"orphan.found", tags)
 }
 
-// MetricWatcherCalls is a counter of watcher calls.
+// MetricWatcherCalls is a counter of watcher calls. This is emitted by every OnChange event for all of our watchers,
+// e.g., the chaos pod watcher, the target pod watcher, the disruption watcher.
 func (d Sink) MetricWatcherCalls(tags []string) error {
 	return d.metricWithStatus(d.prefix+"watcher.calls_total", tags)
 }
 
-// MetricTooLate reports when a scheduled disruption misses its aloted time to be scheduled
+// MetricTooLate reports when a scheduled Disruption misses its configured time to be run,
 // specific to cron and rollout controllers
 func (d Sink) MetricTooLate(tags []string) error {
 	return d.metricWithStatus(d.prefix+"schedule.too_late", tags)
 }
 
-// MetricTargetMissing reports when a scheduled Disruption can not find its specific target
-// either for the first time or multiple times. A deletion occurs on the final alert
+// MetricTargetMissing reports anytime scheduled Disruption can not find its specified target
 func (d Sink) MetricTargetMissing(duration time.Duration, tags []string) error {
 	return d.timing(d.prefix+"schedule.target_missing", duration, tags)
 }
 
-// MetricMissingTargetFound reports when a scheduled Disruption which had initially been deemed missing
+// MetricMissingTargetFound reports when a scheduled Disruption's target which had initially been deemed missing
 // is "found" and running in the kubernetes namespace
 func (d Sink) MetricMissingTargetFound(tags []string) error {
 	return d.metricWithStatus(d.prefix+"schedule.missing_target_found", tags)
@@ -243,7 +260,7 @@ func (d Sink) MetricMissingTargetDeleted(tags []string) error {
 	return d.metricWithStatus(d.prefix+"schedule.missing_target_deleted", tags)
 }
 
-// MetricNextScheduledTime reports the duration until the next scheduled disruption will run
+// MetricNextScheduledTime reports the duration until this scheduled Disruption's next scheduled disruption should run
 func (d Sink) MetricNextScheduledTime(duration time.Duration, tags []string) error {
 	return d.timing(d.prefix+"schedule.next_scheduled", duration, tags)
 }