Added cert-controller for easy webhook cert generation

Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>

Co-authored-by: Tomoki Sugiura <tomoki.sugiura@mail.shanpu.info>

Author: p-strusiewiczsurmacki-mobica

.github/workflows/ci.yaml

@@ -89,3 +89,32 @@ jobs:
       with:
         name: logs-ipv6-${{ matrix.ipv6 }}-with-ipam-${{ matrix.with-ipam }}-${{ matrix.kindest-node }}.tar.gz
         path: v2/e2e/logs.tar.gz
+  certs-generation:
+    name: Cert generation test
+    strategy:
+      matrix:
+        kindest-node: ["1.29.12", "1.30.8", "1.31.4"]
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.go-version }}
+        cache-dependency-path: "**/go.sum"
+    - run: make image
+    - run: make enable-certs-generation
+      working-directory: v2/e2e
+    - run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }}
+      working-directory: v2/e2e
+    - run: make install-coil
+      working-directory: v2/e2e
+    - run: make test
+      working-directory: v2/e2e
+    - run: make logs
+      working-directory: v2/e2e
+      if: always()
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: logs-cert-generation-${{ matrix.kindest-node }}.tar.gz
+        path: v2/e2e/logs.tar.gz

docs/setup.md

@@ -8,7 +8,9 @@ The YAML manifests of Coil can be generated using [kustomize](https://kubernetes
 You can tweak optional parameters by editing [`kustomization.yaml`](../v2/kustomization.yaml) file.
 
 - [Install `kustomize`](#install-kustomize)
-- [Generate TLS certificate](#generate-tls-certificate)
+- [TLS certificates](#tls-certificates)
+  - [Generate certificates manually](#generate-certificates-manually)
+  - [Enable automatic certs generation](#enable-automatic-certs-generation)
 - [Edit `kustomization.yaml`](#edit-kustomizationyaml)
 - [Edit `netconf.json`](#edit-netconfjson)
 - [Compile and apply the manifest](#compile-and-apply-the-manifest)
@@ -19,17 +21,23 @@ You can tweak optional parameters by editing [`kustomization.yaml`](../v2/kustom
 - [Note on CRI runtime compatibility](#note-on-cri-runtime-compatibility)
 - [Standalone egress](#standalone-egress)
   - [Configuration](#configuration)
+  - [Testing standalone egress](#testing-standalone-egress)
+    - [Testing with Kindnet using IPv4](#testing-with-kindnet-using-ipv4)
+    - [Testing with Kindnet using IPv6](#testing-with-kindnet-using-ipv6)
 
 ## Install `kustomize`
 
 Follow the instructions: https://kubectl.docs.kubernetes.io/installation/kustomize/
 
 `kustomize` 4.1.3 is verified to work for Coil.
 
-## Generate TLS certificate
+## TLS certificates
 
-Coil runs an admission webhook server, and it needs a self-signed certificate.
-Run `make certs` under `v2/` directory to generate the certificate.
+Coil runs an admission webhook server, and it needs a self-signed certificate. You can either generate those manually or `coil` can generate those for you upon start.
+
+### Generate certificates manually
+
+Run `make certs` under `v2/` directory to generate the certificates.
 
 ```console
 $ make certs
@@ -43,6 +51,16 @@ config/default/cert.pem         config/default/egress-key.pem  config/default/ip
 config/default/egress-cert.pem  config/default/ipam-cert.pem   config/default/key.pem
 ```
 
+### Enable automatic certs generation
+
+Run `make enable-certs-generation` under `v2/` directory to enable automatic certificate generation in `coil`.
+
+```console
+$ make enable-certs-generation
+```
+
+This will configure the kustomization files that will can be later used by `make install-coil` target.
+
 ## Edit `kustomization.yaml`
 
 `kustomization.yaml` under `v2/` directory contains some commented option settings.

v2/Makefile

@@ -75,7 +75,7 @@ check-generate:
 	$(MAKE) generate
 	$(MAKE) manifests
 	go mod tidy
-	git diff --exit-code
+	git diff --exit-code -- ':!config/rbac/coil-egress-controller_role.yaml' ':!config/rbac/coil-ipam-controller_role.yaml'
 
 # Generate manifests e.g. CRD, RBAC etc.
 .PHONY: manifests
@@ -110,7 +110,8 @@ $(YQ):
 COIL_IPAM_CONTROLLER_ROLE_DEPENDS = controllers/addresspool_controller.go \
 	controllers/blockrequest_controller.go \
 	pkg/ipam/pool.go \
-	runners/garbage_collector.go
+	runners/garbage_collector.go \
+	pkg/cert/cert.go
 
 config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS)
 	-rm -rf work
@@ -119,19 +120,22 @@ config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS)
 	sed '0,/^package/s/.*/package work/' controllers/blockrequest_controller.go > work/blockrequest_controller.go
 	sed '0,/^package/s/.*/package work/' pkg/ipam/pool.go > work/pool.go
 	sed '0,/^package/s/.*/package work/' runners/garbage_collector.go > work/garbage_collector.go
+	sed '0,/^package/s/.*/package work/' pkg/cert/cert.go > work/cert.go
 	$(CONTROLLER_GEN) rbac:roleName=coil-ipam-controller paths=./work output:stdout > $@
 	rm -rf work
 
 COIL_EGRESS_CONTROLLER_ROLE_DEPENDS = controllers/egress_controller.go \
-	controllers/clusterrolebinding_controller.go
+	controllers/clusterrolebinding_controller.go \
+	pkg/cert/cert.go
 
 config/rbac/coil-egress-controller_role.yaml: $(COIL_EGRESS_CONTROLLER_ROLE_DEPENDS)
 	-rm -rf work
 	mkdir work
 	sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go
 	sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go
+	sed '0,/^package/s/.*/package work/' pkg/cert/cert.go > work/cert.go
 	$(CONTROLLER_GEN) rbac:roleName=coil-egress-controller paths=./work output:stdout > $@
-	# rm -rf work
+	rm -rf work
 
 COILD_DEPENDS = controllers/blockrequest_watcher.go \
 	pkg/ipam/node.go \
@@ -259,3 +263,19 @@ staticcheck:
 	if ! which staticcheck >/dev/null; then \
 		env GOFLAGS= go install honnef.co/go/tools/cmd/staticcheck@latest; \
 	fi
+
+define comment_certs
+    $(eval $@_FILE = $(1))
+	sed -i -E "{s/(^patchesStrategicMerge.*)/# \1/}" ${$@_FILE}
+	sed -i -E "{s/(.*webhook_manifests_patch.*)/# \1/}" ${$@_FILE}
+	sed -i -E "{s/(.*files.*)/# \1/g}" ${$@_FILE}
+	sed -i -E "{s/(.*\.pem.*)/# \1/g}" ${$@_FILE}
+	sed -i -E "{s/(.*\/tls.*)/# \1/g}" ${$@_FILE}
+endef    
+
+.PHONY: enable-certs-generation
+enable-certs-generation:
+	sed -i "22,47 {s/^# //}" kustomization.yaml
+	@$(call comment_certs,"config/default/kustomization.yaml")
+	@$(call comment_certs,"config/default/egress/v4/kustomization.yaml")
+	@$(call comment_certs,"config/default/egress/v6/kustomization.yaml")

v2/cmd/coil-egress-controller/sub/root.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	v2 "github.com/cybozu-go/coil/v2"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -18,6 +19,9 @@ var config struct {
 	certDir     string
 	egressPort  int32
 	zapOpts     zap.Options
+
+	enableCertRotation         bool
+	enableRestartOnCertRefresh bool
 }
 
 var rootCmd = &cobra.Command{
@@ -47,6 +51,8 @@ func init() {
 	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9444", "bind address of admission webhook")
 	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
 	pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress")
+	pf.BoolVar(&config.enableCertRotation, "enable-cert-rotation", constants.DefaultEnableCertRotation, "enables webhook's certificate generation")
+	pf.BoolVar(&config.enableRestartOnCertRefresh, "enable-restart-on-cert-refresh", constants.DefaultEnableRestartOnCertRefresh, "enables pod's restart on webhook certificate refresh")
 
 	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(goflags)

v2/cmd/coil-egress-controller/sub/run.go

@@ -1,6 +1,7 @@
 package sub
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os"
@@ -10,6 +11,7 @@ import (
 	v2 "github.com/cybozu-go/coil/v2"
 	coilv2 "github.com/cybozu-go/coil/v2/api/v2"
 	"github.com/cybozu-go/coil/v2/controllers"
+	"github.com/cybozu-go/coil/v2/pkg/cert"
 	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -78,9 +80,41 @@ func subMain() error {
 		return err
 	}
 
-	// register controllers
+	setupFinished := make(chan struct{})
+
+	if config.enableCertRotation {
+		setupFinished, err = cert.SetupRotator(mgr, "egress", config.enableRestartOnCertRefresh, setupFinished)
+		if err != nil {
+			return fmt.Errorf("failed to setup Rotator: %w", err)
+		}
+	} else {
+		close(setupFinished)
+	}
+
+	setupErr := make(chan error)
+
+	go func() {
+		setupErr <- setupManager(mgr, setupFinished)
+		close(setupErr)
+	}()
+
+	mgrCtx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
+	defer cancel()
+
+	mgrErr := make(chan error)
+	go func() {
+		setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
+		if err := mgr.Start(mgrCtx); err != nil {
+			mgrErr <- err
+		}
+		close(mgrErr)
+	}()
+
+	return cert.WaitForExit(setupErr, mgrErr, cancel)
+}
 
-	ctx := ctrl.SetupSignalHandler()
+func setupManager(mgr ctrl.Manager, certCompleted chan struct{}) error {
+	// register controllers
 
 	podNS := os.Getenv(constants.EnvPodNamespace)
 	podName := os.Getenv(constants.EnvPodName)
@@ -102,19 +136,13 @@ func subMain() error {
 		return err
 	}
 
-	// register webhooks
+	// wait for certificates to be configured
+	<-certCompleted
 
+	// register webhooks
 	if err := (&coilv2.Egress{}).SetupWebhookWithManager(mgr); err != nil {
 		return err
 	}
 
-	// start manager
-
-	setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
-		return err
-	}
-
 	return nil
 }

v2/cmd/coil-ipam-controller/sub/root.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	v2 "github.com/cybozu-go/coil/v2"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -19,6 +20,9 @@ var config struct {
 	certDir     string
 	gcInterval  time.Duration
 	zapOpts     zap.Options
+
+	enableCertRotation         bool
+	enableRestartOnCertRefresh bool
 }
 
 var rootCmd = &cobra.Command{
@@ -48,6 +52,8 @@ func init() {
 	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9443", "bind address of admission webhook")
 	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
 	pf.DurationVar(&config.gcInterval, "gc-interval", 1*time.Hour, "garbage collection interval")
+	pf.BoolVar(&config.enableCertRotation, "enable-cert-rotation", constants.DefaultEnableCertRotation, "enables webhook's certificate generation")
+	pf.BoolVar(&config.enableRestartOnCertRefresh, "enable-restart-on-cert-refresh", constants.DefaultEnableRestartOnCertRefresh, "enables pod's restart on webhook certificate refresh")
 
 	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(goflags)

v2/cmd/coil-ipam-controller/sub/run.go

@@ -1,6 +1,7 @@
 package sub
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"strconv"
@@ -9,6 +10,7 @@ import (
 	v2 "github.com/cybozu-go/coil/v2"
 	coilv2 "github.com/cybozu-go/coil/v2/api/v2"
 	"github.com/cybozu-go/coil/v2/controllers"
+	"github.com/cybozu-go/coil/v2/pkg/cert"
 	"github.com/cybozu-go/coil/v2/pkg/indexing"
 	"github.com/cybozu-go/coil/v2/pkg/ipam"
 	"github.com/cybozu-go/coil/v2/runners"
@@ -78,6 +80,41 @@ func subMain() error {
 		return err
 	}
 
+	setupFinished := make(chan struct{})
+
+	if config.enableCertRotation {
+		setupFinished, err = cert.SetupRotator(mgr, "ipam", config.enableRestartOnCertRefresh, setupFinished)
+		if err != nil {
+			return fmt.Errorf("failed to setup Rotator: %w", err)
+		}
+	} else {
+		close(setupFinished)
+	}
+
+	ctx := ctrl.SetupSignalHandler()
+
+	setupErr := make(chan error)
+
+	go func() {
+		setupErr <- setupManager(ctx, mgr, setupFinished)
+		close(setupErr)
+	}()
+
+	mgrCtx, cancel := context.WithCancel(ctx)
+
+	mgrErr := make(chan error)
+	go func() {
+		setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
+		if err := mgr.Start(mgrCtx); err != nil {
+			mgrErr <- err
+		}
+		close(mgrErr)
+	}()
+
+	return cert.WaitForExit(setupErr, mgrErr, cancel)
+}
+
+func setupManager(ctx context.Context, mgr ctrl.Manager, certCompleted chan struct{}) error {
 	// register controllers
 
 	pm := ipam.NewPoolManager(mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log.WithName("pool-manager"), scheme)
@@ -90,7 +127,6 @@ func subMain() error {
 		return err
 	}
 
-	ctx := ctrl.SetupSignalHandler()
 	if err := indexing.SetupIndexForAddressBlock(ctx, mgr); err != nil {
 		return err
 	}
@@ -104,6 +140,9 @@ func subMain() error {
 		return err
 	}
 
+	// wait for certificates to be configured
+	<-certCompleted
+
 	// register webhooks
 
 	if err := (&coilv2.AddressPool{}).SetupWebhookWithManager(mgr); err != nil {
@@ -117,11 +156,5 @@ func subMain() error {
 		return err
 	}
 
-	setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
-		return err
-	}
-
 	return nil
 }

v2/config/default/egress/v4/kustomization.yaml

@@ -4,6 +4,7 @@ resources:
 - ../../../pod/egress/v4
 - ../../../webhook/egress
 
+# [CERTS] Following lines should be commented if automatic cert generation is used.
 patchesStrategicMerge:
 - ../webhook_manifests_patch.yaml
 
@@ -13,8 +14,9 @@ generatorOptions:
 secretGenerator:
 # [EGRESS] Following lines be uncommented to enable Egress NAT features.
 - name: coilv2-egress-webhook-server-cert
+# [CERTS] Following lines should be commented if automatic cert generation is used.
   files:
   - ca.crt=../../cert.pem
   - tls.crt=../../egress-cert.pem
   - tls.key=../../egress-key.pem
-  type: "kubernetes.io/tls"
+  type: "kubernetes.io/tls"