Added cert-controller for easy webhook cert generation

Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>

Author: p-strusiewiczsurmacki-mobica

README.md

@@ -74,7 +74,6 @@ Prepare a recent Ubuntu and install Docker and Go, then run:
 
 ```console
 $ cd v2
-$ make certs
 $ make image
 
 $ cd e2e

docs/setup.md

@@ -26,10 +26,10 @@ Follow the instructions: https://kubectl.docs.kubernetes.io/installation/kustomi
 
 `kustomize` 4.1.3 is verified to work for Coil.
 
-## Generate TLS certificate
+## Generate TLS certificate (optional)
 
 Coil runs an admission webhook server, and it needs a self-signed certificate.
-Run `make certs` under `v2/` directory to generate the certificate.
+Caoil will generate certificates upon start, but you can also pre-generate those if you want with `make certs` command under `v2/` directory.
 
 ```console
 $ make certs
@@ -275,13 +275,9 @@ To deploy Coil with only egress feature enabled the following changes are requir
 ### Testing standalone egress
 
 #### Testing with Kindnet using IPv4
-1. Generate certificates using `v2/Makefile`.
-    ```bash
-    cd v2 && make certs
-    ```
 1. Go to `v2/e2e`
     ```bash
-    cd e2e
+    cd v2/e2e
     ```
 1. Create IPv4 based Kind cluster with Kindnet CNI deployed:
     ```bash
@@ -297,13 +293,9 @@ To deploy Coil with only egress feature enabled the following changes are requir
     ```
 
 #### Testing with Kindnet using IPv6
-1. Generate certificates using `v2/Makefile`.
-    ```bash
-    cd v2 && make certs
-    ```
 1. Go to `v2/e2e`
     ```bash
-    cd e2e
+    cd v2/e2e
     ```
 1. Create IPv6 based Kind cluster with Kindnet CNI deployed:
     ```bash

v2/Makefile

@@ -110,7 +110,8 @@ $(YQ):
 COIL_IPAM_CONTROLLER_ROLE_DEPENDS = controllers/addresspool_controller.go \
 	controllers/blockrequest_controller.go \
 	pkg/ipam/pool.go \
-	runners/garbage_collector.go
+	runners/garbage_collector.go \
+	pkg/certs/certs.go
 
 config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS)
 	-rm -rf work
@@ -119,19 +120,22 @@ config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS)
 	sed '0,/^package/s/.*/package work/' controllers/blockrequest_controller.go > work/blockrequest_controller.go
 	sed '0,/^package/s/.*/package work/' pkg/ipam/pool.go > work/pool.go
 	sed '0,/^package/s/.*/package work/' runners/garbage_collector.go > work/garbage_collector.go
+	sed '0,/^package/s/.*/package work/' pkg/certs/certs.go > work/certs.go
 	$(CONTROLLER_GEN) rbac:roleName=coil-ipam-controller paths=./work output:stdout > $@
 	rm -rf work
 
 COIL_EGRESS_CONTROLLER_ROLE_DEPENDS = controllers/egress_controller.go \
-	controllers/clusterrolebinding_controller.go
+	controllers/clusterrolebinding_controller.go \
+	pkg/certs/certs.go
 
 config/rbac/coil-egress-controller_role.yaml: $(COIL_EGRESS_CONTROLLER_ROLE_DEPENDS)
 	-rm -rf work
 	mkdir work
 	sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go
 	sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go
+	sed '0,/^package/s/.*/package work/' pkg/certs/certs.go > work/certs.go
 	$(CONTROLLER_GEN) rbac:roleName=coil-egress-controller paths=./work output:stdout > $@
-	# rm -rf work
+	rm -rf work
 
 COILD_DEPENDS = controllers/blockrequest_watcher.go \
 	pkg/ipam/node.go \

v2/cmd/coil-egress-controller/sub/root.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	v2 "github.com/cybozu-go/coil/v2"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -18,6 +19,9 @@ var config struct {
 	certDir     string
 	egressPort  int32
 	zapOpts     zap.Options
+
+	disableCertRotation         bool
+	disableRestartOnCertRefresh bool
 }
 
 var rootCmd = &cobra.Command{
@@ -47,6 +51,8 @@ func init() {
 	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9444", "bind address of admission webhook")
 	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
 	pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress")
+	pf.BoolVar(&config.disableCertRotation, "disable-cert-rotation", constants.DefaultDisableCertRotation, "disables webhook's certificate generation")
+	pf.BoolVar(&config.disableRestartOnCertRefresh, "disable-restart-on-cert-refresh", constants.DefailtDisableRestartOnCertRefresh, "disables pod's restart on webhook certificate refresh")
 
 	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(goflags)

v2/cmd/coil-egress-controller/sub/run.go

@@ -1,6 +1,7 @@
 package sub
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os"
@@ -11,6 +12,7 @@ import (
 	coilv2 "github.com/cybozu-go/coil/v2/api/v2"
 	"github.com/cybozu-go/coil/v2/controllers"
 	"github.com/cybozu-go/coil/v2/pkg/constants"
+	"github.com/cybozu-go/coil/v2/pkg/utils"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -78,9 +80,46 @@ func subMain() error {
 		return err
 	}
 
-	// register controllers
+	var setupFinished chan struct{}
+
+	if !config.disableCertRotation {
+		setupFinished, err = utils.SetupRotator(mgr, "egress", config.disableRestartOnCertRefresh)
+		if err != nil {
+			return fmt.Errorf("failed to setup Rotator: %w", err)
+		}
+	}
+
+	setupErr := make(chan error)
+
+	go func() {
+		setupErr <- setupManager(mgr, setupFinished)
+		close(setupErr)
+	}()
 
-	ctx := ctrl.SetupSignalHandler()
+	mgrCtx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
+	defer cancel()
+
+	mgrErr := make(chan error)
+	go func() {
+		setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
+		if err := mgr.Start(mgrCtx); err != nil {
+			mgrErr <- err
+		}
+		close(mgrErr)
+	}()
+
+	if err := utils.WaitForExit(setupErr, mgrErr, cancel); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func setupManager(mgr ctrl.Manager, setupFinished chan struct{}) error {
+	// wait for certificates to be configured
+	<-setupFinished
+
+	// register controllers
 
 	podNS := os.Getenv(constants.EnvPodNamespace)
 	podName := os.Getenv(constants.EnvPodName)
@@ -108,13 +147,5 @@ func subMain() error {
 		return err
 	}
 
-	// start manager
-
-	setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
-		return err
-	}
-
 	return nil
 }

v2/cmd/coil-ipam-controller/sub/root.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	v2 "github.com/cybozu-go/coil/v2"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -19,6 +20,9 @@ var config struct {
 	certDir     string
 	gcInterval  time.Duration
 	zapOpts     zap.Options
+
+	disableCertRotation         bool
+	disableRestartOnCertRefresh bool
 }
 
 var rootCmd = &cobra.Command{
@@ -48,6 +52,8 @@ func init() {
 	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9443", "bind address of admission webhook")
 	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
 	pf.DurationVar(&config.gcInterval, "gc-interval", 1*time.Hour, "garbage collection interval")
+	pf.BoolVar(&config.disableCertRotation, "disable-cert-rotation", constants.DefaultDisableCertRotation, "disables webhook's certificate generation")
+	pf.BoolVar(&config.disableRestartOnCertRefresh, "disable-restart-on-cert-refresh", constants.DefailtDisableRestartOnCertRefresh, "disables pod's restart on webhook certificate refresh")
 
 	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(goflags)

v2/cmd/coil-ipam-controller/sub/run.go

@@ -1,6 +1,7 @@
 package sub
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strconv"
@@ -11,6 +12,7 @@ import (
 	"github.com/cybozu-go/coil/v2/controllers"
 	"github.com/cybozu-go/coil/v2/pkg/indexing"
 	"github.com/cybozu-go/coil/v2/pkg/ipam"
+	"github.com/cybozu-go/coil/v2/pkg/utils"
 	"github.com/cybozu-go/coil/v2/runners"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -78,6 +80,46 @@ func subMain() error {
 		return err
 	}
 
+	var setupFinished chan struct{}
+
+	if !config.disableCertRotation {
+		setupFinished, err = utils.SetupRotator(mgr, "ipam", config.disableRestartOnCertRefresh)
+		if err != nil {
+			return fmt.Errorf("failed to setup Rotator: %w", err)
+		}
+	}
+
+	ctx := ctrl.SetupSignalHandler()
+
+	setupErr := make(chan error)
+
+	go func() {
+		setupErr <- setupManager(ctx, mgr, setupFinished)
+		close(setupErr)
+	}()
+
+	mgrCtx, cancel := context.WithCancel(ctx)
+
+	mgrErr := make(chan error)
+	go func() {
+		setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
+		if err := mgr.Start(mgrCtx); err != nil {
+			mgrErr <- err
+		}
+		close(mgrErr)
+	}()
+
+	if err := utils.WaitForExit(setupErr, mgrErr, cancel); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func setupManager(ctx context.Context, mgr ctrl.Manager, setupFinished chan struct{}) error {
+	// wait for certificates to be configured
+	<-setupFinished
+
 	// register controllers
 
 	pm := ipam.NewPoolManager(mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log.WithName("pool-manager"), scheme)
@@ -90,7 +132,6 @@ func subMain() error {
 		return err
 	}
 
-	ctx := ctrl.SetupSignalHandler()
 	if err := indexing.SetupIndexForAddressBlock(ctx, mgr); err != nil {
 		return err
 	}
@@ -117,11 +158,5 @@ func subMain() error {
 		return err
 	}
 
-	setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
-		return err
-	}
-
 	return nil
 }

v2/config/default/egress/v4/kustomization.yaml

@@ -4,17 +4,19 @@ resources:
 - ../../../pod/egress/v4
 - ../../../webhook/egress
 
-patchesStrategicMerge:
-- ../webhook_manifests_patch.yaml
+# [CERTS] Following lines should be uncommented if pre-generated certs are used.
+# patchesStrategicMerge:
+# - ../webhook_manifests_patch.yaml
 
 generatorOptions:
   disableNameSuffixHash: true
 
 secretGenerator:
 # [EGRESS] Following lines be uncommented to enable Egress NAT features.
 - name: coilv2-egress-webhook-server-cert
-  files:
-  - ca.crt=../../cert.pem
-  - tls.crt=../../egress-cert.pem
-  - tls.key=../../egress-key.pem
-  type: "kubernetes.io/tls"
+# [CERTS] Following lines should be uncommented if pre-generated certs are used.
+  # files:
+  # - ca.crt=../../cert.pem
+  # - tls.crt=../../egress-cert.pem
+  # - tls.key=../../egress-key.pem
+  # type: "kubernetes.io/tls"

v2/config/default/egress/v6/kustomization.yaml

@@ -4,17 +4,19 @@ resources:
 - ../../../pod/egress/v6
 - ../../../webhook/egress
 
-patchesStrategicMerge:
-- ../webhook_manifests_patch.yaml
+# [CERTS] Following lines should be uncommented if pre-generated certs are used.
+# patchesStrategicMerge:
+# - ../webhook_manifests_patch.yaml
 
 generatorOptions:
   disableNameSuffixHash: true
 
 secretGenerator:
 # [EGRESS] Following lines be uncommented to enable Egress NAT features.
 - name: coilv2-egress-webhook-server-cert
-  files:
-  - ca.crt=../../cert.pem
-  - tls.crt=../../egress-cert.pem
-  - tls.key=../../egress-key.pem
-  type: "kubernetes.io/tls"
+# [CERTS] Following lines should be uncommented if pre-generated certs are used.
+  # files:
+  # - ca.crt=../../cert.pem
+  # - tls.crt=../../egress-cert.pem
+  # - tls.key=../../egress-key.pem
+  # type: "kubernetes.io/tls"