cybozu-go · chez-shanpu · Apr 4, 2025 · Mar 10, 2025
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -89,3 +89,32 @@ jobs:
       with:
         name: logs-ipv6-${{ matrix.ipv6 }}-with-ipam-${{ matrix.with-ipam }}-${{ matrix.kindest-node }}.tar.gz
         path: v2/e2e/logs.tar.gz
+  certs-generation:
+    name: Cert generation test
+    strategy:
+      matrix:
+        kindest-node: ["1.29.12", "1.30.8", "1.31.4"]
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v5
+      with:
+        go-version: ${{ env.go-version }}
+        cache-dependency-path: "**/go.sum"
+    - run: make image
+    - run: make enable-certs-rotation
+      working-directory: v2/e2e
+    - run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }}
+      working-directory: v2/e2e
+    - run: make install-coil
+      working-directory: v2/e2e
+    - run: make test
+      working-directory: v2/e2e
+    - run: make logs
+      working-directory: v2/e2e
+      if: always()
+    - uses: actions/upload-artifact@v4
+      if: always()
+      with:
+        name: logs-cert-generation-${{ matrix.kindest-node }}.tar.gz
+        path: v2/e2e/logs.tar.gz
diff --git a/docs/cmd-coil-egress-controller.md b/docs/cmd-coil-egress-controller.md
@@ -16,11 +16,13 @@ and binds it to the **ClusterRoles** for `coil-egress`.
 
 ```
 Flags:
-      --cert-dir string        directory to locate TLS certs for webhook (default "/certs")
-      --egress-port int32      UDP port number used by coil-egress (default 5555)
-      --health-addr string     bind address of health/readiness probes (default ":9387")
-  -h, --help                   help for coil-egress-controller
-      --metrics-addr string    bind address of metrics endpoint (default ":9386")
-  -v, --version                version for coil-egress-controller
-      --webhook-addr string    bind address of admission webhook (default ":9443")
+      --cert-dir string                 directory to locate TLS certs for webhook (default "/certs")
+      --egress-port int32               UDP port number used by coil-egress (default 5555)
+      --health-addr string              bind address of health/readiness probes (default ":9387")
+  -h, --help                            help for coil-egress-controller
+      --metrics-addr string             bind address of metrics endpoint (default ":9386")
+  -v, --version                         version for coil-egress-controller
+      --webhook-addr string             bind address of admission webhook (default ":9443")
+      --enable-cert-rotation            enables webhook's certificate generation
+      --enable-restart-on-cert-refresh  enables pod's restart on webhook certificate refresh
 ```
diff --git a/docs/cmd-coil-ipam-controller.md b/docs/cmd-coil-ipam-controller.md
@@ -22,13 +22,15 @@ address blocks from the requested pool.
 
 ```
 Flags:
-      --cert-dir string        directory to locate TLS certs for webhook (default "/certs")
-      --gc-interval duration   garbage collection interval (default 1h0m0s)
-      --health-addr string     bind address of health/readiness probes (default ":9387")
-  -h, --help                   help for coil-ipam-controller
-      --metrics-addr string    bind address of metrics endpoint (default ":9386")
-  -v, --version                version for coil-ipam-controller
-      --webhook-addr string    bind address of admission webhook (default ":9443")
+      --cert-dir string                 directory to locate TLS certs for webhook (default "/certs")
+      --gc-interval duration            garbage collection interval (default 1h0m0s)
+      --health-addr string              bind address of health/readiness probes (default ":9387")
+  -h, --help                            help for coil-ipam-controller
+      --metrics-addr string             bind address of metrics endpoint (default ":9386")
+  -v, --version                         version for coil-ipam-controller
+      --webhook-addr string             bind address of admission webhook (default ":9443")
+      --enable-cert-rotation            enables webhook's certificate generation
+      --enable-restart-on-cert-refresh  enables pod's restart on webhook certificate refresh
 ```
 
 ## Prometheus metrics

diff --git a/docs/setup.md b/docs/setup.md
@@ -8,7 +8,9 @@ The YAML manifests of Coil can be generated using [kustomize](https://kubernetes
 You can tweak optional parameters by editing [`kustomization.yaml`](../v2/kustomization.yaml) file.
 
 - [Install `kustomize`](#install-kustomize)
-- [Generate TLS certificate](#generate-tls-certificate)
+- [TLS certificates](#tls-certificates)
+  - [Generate certificates manually](#generate-certificates-manually)
+  - [Enable automatic certs rotation](#enable-automatic-certs-rotation)
 - [Edit `kustomization.yaml`](#edit-kustomizationyaml)
 - [Edit `netconf.json`](#edit-netconfjson)
 - [Compile and apply the manifest](#compile-and-apply-the-manifest)
@@ -19,17 +21,23 @@ You can tweak optional parameters by editing [`kustomization.yaml`](../v2/kustom
 - [Note on CRI runtime compatibility](#note-on-cri-runtime-compatibility)
 - [Standalone egress](#standalone-egress)
   - [Configuration](#configuration)
+  - [Testing standalone egress](#testing-standalone-egress)
+    - [Testing with Kindnet using IPv4](#testing-with-kindnet-using-ipv4)
+    - [Testing with Kindnet using IPv6](#testing-with-kindnet-using-ipv6)
 
 ## Install `kustomize`
 
 Follow the instructions: https://kubectl.docs.kubernetes.io/installation/kustomize/
 
 `kustomize` 4.1.3 is verified to work for Coil.
 
-## Generate TLS certificate
+## TLS certificates
 
-Coil runs an admission webhook server, and it needs a self-signed certificate.
-Run `make certs` under `v2/` directory to generate the certificate.
+Coil runs admission webhook servers, and each one needs a self-signed certificate. You can either generate certificates manually or have Coil create them when it starts up.
+
+### Generate certificates manually
+
+Run `make certs` under `v2/` directory to generate the certificates.
 
 ```console
 $ make certs
@@ -43,6 +51,16 @@ config/default/cert.pem         config/default/egress-key.pem  config/default/ip
 config/default/egress-cert.pem  config/default/ipam-cert.pem   config/default/key.pem
 ```
 
+### Enable automatic certs rotation
+
+Run `make enable-certs-rotation` under `v2/` directory to enable automatic certificate generation in `coil`.
+
+```console
+$ make enable-certs-rotation
+```
+
+This will configure the kustomization files that will can be later used by `make install-coil` target.
+
 ## Edit `kustomization.yaml`
 
 `kustomization.yaml` under `v2/` directory contains some commented option settings.

diff --git a/v2/Makefile b/v2/Makefile
@@ -16,7 +16,9 @@ SETUP_ENVTEST := $(shell pwd)/bin/setup-envtest
 YQ := $(shell pwd)/bin/yq
 CRD_OPTIONS = "crd:crdVersions=v1"
 ROLES = config/rbac/coil-ipam-controller_role.yaml \
+	config/rbac/coil-ipam-controller-certs_role.yaml \
 	config/rbac/coil-egress-controller_role.yaml \
+	config/rbac/coil-egress-controller-certs_role.yaml \
 	config/rbac/coild_role.yaml \
 	config/rbac/coil-router_role.yaml \
 	config/rbac/coil-egress_role.yaml
@@ -122,6 +124,23 @@ config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS)
 	$(CONTROLLER_GEN) rbac:roleName=coil-ipam-controller paths=./work output:stdout > $@
 	rm -rf work
 
+COIL_IPAM_CONTROLLER_CERTS_ROLE_DEPENDS = controllers/addresspool_controller.go \
+	controllers/blockrequest_controller.go \
+	pkg/ipam/pool.go \
+	runners/garbage_collector.go \
+	pkg/cert/cert.go
+
+config/rbac/coil-ipam-controller-certs_role.yaml: $(COIL_IPAM_CONTROLLER_CERTS_ROLE_DEPENDS)
+	-rm -rf work
+	mkdir work
+	sed '0,/^package/s/.*/package work/' controllers/addresspool_controller.go > work/addresspool_controller.go
+	sed '0,/^package/s/.*/package work/' controllers/blockrequest_controller.go > work/blockrequest_controller.go
+	sed '0,/^package/s/.*/package work/' pkg/ipam/pool.go > work/pool.go
+	sed '0,/^package/s/.*/package work/' runners/garbage_collector.go > work/garbage_collector.go
+	sed '0,/^package/s/.*/package work/' pkg/cert/cert.go > work/cert.go
+	$(CONTROLLER_GEN) rbac:roleName=coil-ipam-controller paths=./work output:stdout > $@
+	rm -rf work
+
 COIL_EGRESS_CONTROLLER_ROLE_DEPENDS = controllers/egress_controller.go \
 	controllers/clusterrolebinding_controller.go
 
@@ -131,7 +150,20 @@ config/rbac/coil-egress-controller_role.yaml: $(COIL_EGRESS_CONTROLLER_ROLE_DEPE
 	sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go
 	sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go
 	$(CONTROLLER_GEN) rbac:roleName=coil-egress-controller paths=./work output:stdout > $@
-	# rm -rf work
+	rm -rf work
+
+COIL_EGRESS_CONTROLLER_CERTS_ROLE_DEPENDS = controllers/egress_controller.go \
+	controllers/clusterrolebinding_controller.go \
+	pkg/cert/cert.go
+
+config/rbac/coil-egress-controller-certs_role.yaml: $(COIL_EGRESS_CONTROLLER_CERTS_ROLE_DEPENDS)
+	-rm -rf work
+	mkdir work
+	sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go
+	sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go
+	sed '0,/^package/s/.*/package work/' pkg/cert/cert.go > work/cert.go
+	$(CONTROLLER_GEN) rbac:roleName=coil-egress-controller paths=./work output:stdout > $@
+	rm -rf work
 
 COILD_DEPENDS = controllers/blockrequest_watcher.go \
 	pkg/ipam/node.go \
@@ -259,3 +291,23 @@ staticcheck:
 	if ! which staticcheck >/dev/null; then \
 		env GOFLAGS= go install honnef.co/go/tools/cmd/staticcheck@latest; \
 	fi
+
+define comment_certs
+    $(eval $@_FILE = $(1))
+	@sed -i -E "{s/(^patchesStrategicMerge.*)/# \1/}" ${$@_FILE}
+	@sed -i -E "{s/(^-.*webhook_manifests_patch.*)/# \1/}" ${$@_FILE}
+endef    
+
+.PHONY: enable-certs-rotation
+enable-certs-rotation:
+	@sed -i "9,21 {s/^# //}" kustomization.yaml
+	@sed -i -E 's/^(- coil-.*controller_role\.yaml)/# \1/g' ../config/rbac/kustomization.yaml
+	@sed -i -E 's/^# (- coil-.*controller-certs_role\.yaml)/\1/g' ../config/rbac/kustomization.yaml
+	@touch config/default/cert.pem
+	@touch config/default/ipam-cert.pem
+	@touch config/default/ipam-key.pem
+	@touch config/default/egress-cert.pem
+	@touch config/default/egress-key.pem
+	@$(call comment_certs,"config/default/kustomization.yaml")
+	@$(call comment_certs,"config/default/egress/v4/kustomization.yaml")
+	@$(call comment_certs,"config/default/egress/v6/kustomization.yaml")
diff --git a/v2/cmd/coil-egress-controller/sub/root.go b/v2/cmd/coil-egress-controller/sub/root.go
@@ -6,6 +6,7 @@ import (
 	"os"
 
 	v2 "github.com/cybozu-go/coil/v2"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -18,6 +19,9 @@ var config struct {
 	certDir     string
 	egressPort  int32
 	zapOpts     zap.Options
+
+	enableCertRotation         bool
+	enableRestartOnCertRefresh bool
 }
 
 var rootCmd = &cobra.Command{
@@ -47,6 +51,8 @@ func init() {
 	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9444", "bind address of admission webhook")
 	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
 	pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress")
+	pf.BoolVar(&config.enableCertRotation, "enable-cert-rotation", constants.DefaultEnableCertRotation, "enables webhook's certificate generation")
+	pf.BoolVar(&config.enableRestartOnCertRefresh, "enable-restart-on-cert-refresh", constants.DefaultEnableRestartOnCertRefresh, "enables pod's restart on webhook certificate refresh")
 
 	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(goflags)

diff --git a/v2/cmd/coil-egress-controller/sub/run.go b/v2/cmd/coil-egress-controller/sub/run.go
@@ -1,6 +1,7 @@
 package sub
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"os"
@@ -10,6 +11,7 @@ import (
 	v2 "github.com/cybozu-go/coil/v2"
 	coilv2 "github.com/cybozu-go/coil/v2/api/v2"
 	"github.com/cybozu-go/coil/v2/controllers"
+	"github.com/cybozu-go/coil/v2/pkg/cert"
 	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -78,9 +80,40 @@ func subMain() error {
 		return err
 	}
 
-	// register controllers
+	certCompleted := make(chan struct{})
+
+	if config.enableCertRotation {
+		if certCompleted, err = cert.SetupRotator(mgr, "egress", config.enableRestartOnCertRefresh, certCompleted); err != nil {
+			return fmt.Errorf("failed to setup Rotator: %w", err)
+		}
+	} else {
+		close(certCompleted)
+	}
+
+	setupErr := make(chan error)
+
+	go func() {
+		setupErr <- setupManager(mgr, certCompleted)
+		close(setupErr)
+	}()
+
+	mgrCtx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
+	defer cancel()
+
+	mgrErr := make(chan error)
+	go func() {
+		setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
+		if err := mgr.Start(mgrCtx); err != nil {
+			mgrErr <- err
+		}
+		close(mgrErr)
+	}()
+
+	return cert.WaitForExit(setupErr, mgrErr, cancel)
+}
 
-	ctx := ctrl.SetupSignalHandler()
+func setupManager(mgr ctrl.Manager, certCompleted chan struct{}) error {
+	// register controllers
 
 	podNS := os.Getenv(constants.EnvPodNamespace)
 	podName := os.Getenv(constants.EnvPodName)
@@ -102,19 +135,13 @@ func subMain() error {
 		return err
 	}
 
-	// register webhooks
+	// wait for certificates to be configured
+	<-certCompleted
 
+	// register webhooks
 	if err := (&coilv2.Egress{}).SetupWebhookWithManager(mgr); err != nil {
 		return err
 	}
 
-	// start manager
-
-	setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
-		return err
-	}
-
 	return nil
 }
diff --git a/v2/cmd/coil-ipam-controller/sub/root.go b/v2/cmd/coil-ipam-controller/sub/root.go
@@ -7,6 +7,7 @@ import (
 	"time"
 
 	v2 "github.com/cybozu-go/coil/v2"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -19,6 +20,9 @@ var config struct {
 	certDir     string
 	gcInterval  time.Duration
 	zapOpts     zap.Options
+
+	enableCertRotation         bool
+	enableRestartOnCertRefresh bool
 }
 
 var rootCmd = &cobra.Command{
@@ -48,6 +52,8 @@ func init() {
 	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9443", "bind address of admission webhook")
 	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
 	pf.DurationVar(&config.gcInterval, "gc-interval", 1*time.Hour, "garbage collection interval")
+	pf.BoolVar(&config.enableCertRotation, "enable-cert-rotation", constants.DefaultEnableCertRotation, "enables webhook's certificate generation")
+	pf.BoolVar(&config.enableRestartOnCertRefresh, "enable-restart-on-cert-refresh", constants.DefaultEnableRestartOnCertRefresh, "enables pod's restart on webhook certificate refresh")
 
 	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(goflags)