Add a second ConfigMap for individual key overrides (#929)

* Add a second configmap, for specifying overrides as individual keys

Author: ptnapoleon

chart/Chart.yaml

@@ -5,4 +5,4 @@
 apiVersion: v2
 name: chaos-controller
 description: Datadog Chaos Controller chart
-version: 3.1.1
+version: 3.2.0

chart/templates/configmap_overrides.yaml

@@ -0,0 +1,16 @@
+# Unless explicitly stated otherwise all files in this repository are licensed
+# under the Apache License Version 2.0.
+# This product includes software developed at Datadog (https://www.datadoghq.com/).
+# Copyright 2024 Datadog, Inc.
+{{- if .Values.controller.configMapOverrides }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: chaos-controller-overrides
+  namespace: "{{ .Values.chaosNamespace }}"
+data:
+  controller.deleteOnly: "false"
+  # Include any key from the chaos-controller config map's config.yaml
+  # Keys must be flattened and values must be strings, e.g., `controller:\n\tdeleteOnly: true` becomes `controller.deleteOnly: "true"`
+  # This configMap is strictly optional, and primarily used for dynamically altering config from external systems
+{{- end }}

chart/templates/deployment.yaml

@@ -67,6 +67,9 @@ spec:
             - /usr/local/bin/manager
           args:
             - --config=/etc/chaos-controller/config.yaml
+            {{- if .Values.controller.configMapOverrides }}
+            - --config-overrides=chaos-controller-overrides
+            {{- end }}
           env:
             - name: POD_NAMESPACE
               valueFrom:

chart/values.yaml

@@ -16,6 +16,7 @@ global:
       pullSecrets: false # name of the secret object to use when pulling images
 
 controller:
+  configMapOverrides: false # deploy a second configmap, which overrides values from the first
   version: ""
   image:
     repo: chaos-controller

config/config.go

@@ -5,18 +5,22 @@
 package config
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"time"
 
 	cloudtypes "github.com/DataDog/chaos-controller/cloudservice/types"
 	"github.com/DataDog/chaos-controller/eventnotifier"
-	"go.uber.org/zap"
-
+	"github.com/cenkalti/backoff"
 	"github.com/fsnotify/fsnotify"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
+	"go.uber.org/zap"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 )
 
 type config struct {
@@ -109,20 +113,23 @@ type handlerConfig struct {
 const DefaultDisruptionDeletionTimeout = time.Minute * 15
 const DefaultFinalizerDeletionDelay = time.Second * 20
 
-func New(logger *zap.SugaredLogger, osArgs []string) (config, error) {
+func New(client corev1client.ConfigMapInterface, logger *zap.SugaredLogger, osArgs []string) (config, error) {
 	var (
-		configPath string
-		cfg        config
+		configPath         string
+		configMapOverrides string
+		cfg                config
 	)
 
 	preConfigFS := pflag.NewFlagSet("pre-config", pflag.ContinueOnError)
 	mainFS := pflag.NewFlagSet("main-config", pflag.ContinueOnError)
 
 	preConfigFS.ParseErrorsWhitelist.UnknownFlags = true
 	preConfigFS.StringVar(&configPath, "config", "", "Configuration file path")
+	preConfigFS.StringVar(&configMapOverrides, "config-overrides", "", "Name of ConfigMap to provide config overrides")
 	// we redefine configuration flag into main flag to avoid removing it manually from provided args
 	// we also define it to avoid activating "UnknownFlags" for main flags so we'll return an error in case a flag is unknown
 	mainFS.StringVar(&configPath, "config", "", "Configuration file path")
+	mainFS.StringVar(&configMapOverrides, "config-overrides", "", "Name of ConfigMap to provide config overrides")
 
 	mainFS.StringVar(&cfg.Controller.MetricsBindAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 
@@ -544,8 +551,7 @@ func New(logger *zap.SugaredLogger, osArgs []string) (config, error) {
 		return cfg, err
 	}
 
-	err := preConfigFS.Parse(osArgs)
-	if err != nil {
+	if err := preConfigFS.Parse(osArgs); err != nil {
 		return cfg, fmt.Errorf("unable to retrieve configuration parse from provided flag: %w", err)
 	}
 
@@ -559,6 +565,53 @@ func New(logger *zap.SugaredLogger, osArgs []string) (config, error) {
 			return cfg, fmt.Errorf("error loading configuration file: %w", err)
 		}
 
+		if configMapOverrides != "" {
+			var configMap *corev1.ConfigMap
+
+			if backOffErr := backoff.Retry(func() error {
+				var err error
+				ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+				defer cancel()
+				configMap, err = client.Get(ctx, configMapOverrides, metav1.GetOptions{})
+				if err != nil {
+					logger.Debugw(fmt.Sprintf("failed to get %s configMap", configMapOverrides), "error", err)
+				}
+				return err
+			},
+				backoff.NewConstantBackOff(time.Second*5)); backOffErr != nil {
+				return cfg, fmt.Errorf("unable to retry fetching %s: %w", configMapOverrides, backOffErr)
+			}
+
+			interfacedMap := make(map[string]interface{}, len(configMap.Data))
+			for k, v := range configMap.Data {
+				interfacedMap[k] = v
+			}
+
+			if err := viper.MergeConfigMap(interfacedMap); err != nil {
+				return cfg, fmt.Errorf("unable to merge config map: %w", err)
+			}
+
+			go func(resourceVersion string) {
+				ticker := time.NewTicker(time.Second * 30)
+
+				for {
+					<-ticker.C
+
+					configMap, err := client.Get(context.Background(), configMapOverrides, metav1.GetOptions{})
+
+					if err != nil {
+						logger.Errorw(fmt.Sprintf("error getting %s configMap", configMapOverrides), "error", err)
+						continue
+					}
+
+					if configMap.ResourceVersion != resourceVersion {
+						logger.Info("override configmap has changed, restarting")
+						os.Exit(0)
+					}
+				}
+			}(configMap.ResourceVersion)
+		}
+
 		if err := viper.Unmarshal(&cfg); err != nil {
 			return cfg, fmt.Errorf("error unmarshaling configuration: %w", err)
 		}

config/config_test.go

@@ -25,29 +25,29 @@ var _ = Describe("Config", func() {
 
 		Context("invalid config", func() {
 			It("fails with a missing path", func() {
-				_, err := config.New(logger, []string{"--config"})
+				_, err := config.New(nil, logger, []string{"--config"})
 				Expect(err).Should(MatchError("unable to retrieve configuration parse from provided flag: flag needs an argument: --config"))
 			})
 
 			It("fails with an invalid path", func() {
-				_, err := config.New(logger, []string{"--config", "invalid-path/invalid-file.yaml"})
+				_, err := config.New(nil, logger, []string{"--config", "invalid-path/invalid-file.yaml"})
 				Expect(err).Should(MatchError("error loading configuration file: open invalid-path/invalid-file.yaml: no such file or directory"))
 			})
 
 			It("fails with an invalid config file", func() {
-				_, err := config.New(logger, []string{"--config", "testdata/invalid.yaml"})
+				_, err := config.New(nil, logger, []string{"--config", "testdata/invalid.yaml"})
 				Expect(err).Should(MatchError(ContainSubstring("error loading configuration file: While parsing config: yaml: unmarshal errors:")))
 			})
 
 			It("fails with a defaultDuration greater than the maxDuration", func() {
-				_, err := config.New(logger, []string{"--config", "testdata/default-duration-too-big.yaml"})
+				_, err := config.New(nil, logger, []string{"--config", "testdata/default-duration-too-big.yaml"})
 				Expect(err).Should(MatchError(fmt.Sprintf("defaultDuration of %s, must be less than or equal to the maxDuration %s", "3m0s", "2m0s")))
 			})
 		})
 
 		Context("without configuration", func() {
 			It("succeed with default values", func() {
-				v, err := config.New(logger, []string{})
+				v, err := config.New(nil, logger, []string{})
 				Expect(err).ToNot(HaveOccurred())
 
 				By("overriding controller values")
@@ -115,7 +115,7 @@ var _ = Describe("Config", func() {
 
 		Context("with configuration file", func() {
 			It("succeed with overriden values", func() {
-				v, err := config.New(logger, []string{"--config", "testdata/local.yaml"})
+				v, err := config.New(nil, logger, []string{"--config", "testdata/local.yaml"})
 				Expect(err).ToNot(HaveOccurred())
 
 				By("overriding controller values")
@@ -184,7 +184,7 @@ var _ = Describe("Config", func() {
 
 		Context("with configuration file and flag", func() {
 			It("succeed with values from flags", func() {
-				v, err := config.New(logger, []string{"--config", "testdata/local.yaml", "--notifiers-common-clustername", "provided-by-command-flag-cluster-name"})
+				v, err := config.New(nil, logger, []string{"--config", "testdata/local.yaml", "--notifiers-common-clustername", "provided-by-command-flag-cluster-name"})
 				Expect(err).ToNot(HaveOccurred())
 
 				Expect(v.Controller.Notifiers.Common.ClusterName).To(Equal("provided-by-command-flag-cluster-name"))

main.go

@@ -13,6 +13,7 @@ import (
 
 	"go.uber.org/zap"
 	"k8s.io/apimachinery/pkg/api/resource"
+	"k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
 
@@ -86,7 +87,19 @@ func main() {
 		logger.Fatal("missing required CONTROLLER_NODE_NAME environment variable")
 	}
 
-	cfg, err := config.New(logger, os.Args[1:])
+	restConfig, err := rest.InClusterConfig()
+	if err != nil {
+		logger.Fatalw("error creating in-cluster rest config", "error", err)
+	}
+
+	clientset, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		logger.Fatalw("error creating kubernetes clientset", "error", err)
+	}
+
+	configMapClient := clientset.CoreV1().ConfigMaps(os.Getenv("POD_NAMESPACE"))
+
+	cfg, err := config.New(configMapClient, logger, os.Args[1:])
 	if err != nil {
 		logger.Fatalw("unable to create a valid configuration", "error", err)
 	}