Skip to content

Commit fdc44ea

Browse files
p-strusiewiczsurmacki-mobicap-strusiewiczsurmacki-mobica
committed
Added cert-controller for easy webhook cert generation
Signed-off-by: Patryk Strusiewicz-Surmacki <patryk-pawel.strusiewicz-surmacki@external.telekom.de>
1 parent 52b38c6 commit fdc44ea

File tree

18 files changed

+293
-65
lines changed

18 files changed

+293
-65
lines changed

‎README.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -74,7 +74,6 @@ Prepare a recent Ubuntu and install Docker and Go, then run:
7474

7575
```console
7676
$ cd v2
77-
$ make certs
7877
$ make image
7978

8079
$ cd e2e

‎docs/setup.md‎

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ Follow the instructions: https://kubectl.docs.kubernetes.io/installation/kustomi
2626

2727
`kustomize` 4.1.3 is verified to work for Coil.
2828

29-
## Generate TLS certificate
29+
## Generate TLS certificate (optional)
3030

3131
Coil runs an admission webhook server, and it needs a self-signed certificate.
32-
Run `make certs` under `v2/` directory to generate the certificate.
32+
Caoil will generate certificates upon start, but you can also pre-generate those if you want with `make certs` command under `v2/` directory.
3333

3434
```console
3535
$ make certs
@@ -275,13 +275,9 @@ To deploy Coil with only egress feature enabled the following changes are requir
275275
### Testing standalone egress
276276
277277
#### Testing with Kindnet using IPv4
278-
1. Generate certificates using `v2/Makefile`.
279-
```bash
280-
cd v2 && make certs
281-
```
282278
1. Go to `v2/e2e`
283279
```bash
284-
cd e2e
280+
cd v2/e2e
285281
```
286282
1. Create IPv4 based Kind cluster with Kindnet CNI deployed:
287283
```bash
@@ -297,13 +293,9 @@ To deploy Coil with only egress feature enabled the following changes are requir
297293
```
298294
299295
#### Testing with Kindnet using IPv6
300-
1. Generate certificates using `v2/Makefile`.
301-
```bash
302-
cd v2 && make certs
303-
```
304296
1. Go to `v2/e2e`
305297
```bash
306-
cd e2e
298+
cd v2/e2e
307299
```
308300
1. Create IPv6 based Kind cluster with Kindnet CNI deployed:
309301
```bash

‎v2/Makefile‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,8 @@ $(YQ):
110110
COIL_IPAM_CONTROLLER_ROLE_DEPENDS = controllers/addresspool_controller.go \
111111
controllers/blockrequest_controller.go \
112112
pkg/ipam/pool.go \
113-
runners/garbage_collector.go
113+
runners/garbage_collector.go \
114+
pkg/utils/utils.go
114115

115116
config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS)
116117
-rm -rf work
@@ -119,19 +120,22 @@ config/rbac/coil-ipam-controller_role.yaml: $(COIL_IPAM_CONTROLLER_ROLE_DEPENDS)
119120
sed '0,/^package/s/.*/package work/' controllers/blockrequest_controller.go > work/blockrequest_controller.go
120121
sed '0,/^package/s/.*/package work/' pkg/ipam/pool.go > work/pool.go
121122
sed '0,/^package/s/.*/package work/' runners/garbage_collector.go > work/garbage_collector.go
123+
sed '0,/^package/s/.*/package work/' pkg/utils/utils.go > work/utils.go
122124
$(CONTROLLER_GEN) rbac:roleName=coil-ipam-controller paths=./work output:stdout > $@
123125
rm -rf work
124126

125127
COIL_EGRESS_CONTROLLER_ROLE_DEPENDS = controllers/egress_controller.go \
126-
controllers/clusterrolebinding_controller.go
128+
controllers/clusterrolebinding_controller.go \
129+
pkg/utils/utils.go
127130

128131
config/rbac/coil-egress-controller_role.yaml: $(COIL_EGRESS_CONTROLLER_ROLE_DEPENDS)
129132
-rm -rf work
130133
mkdir work
131134
sed '0,/^package/s/.*/package work/' controllers/egress_controller.go > work/egress_controller.go
132135
sed '0,/^package/s/.*/package work/' controllers/clusterrolebinding_controller.go > work/clusterrolebinding_controller.go
136+
sed '0,/^package/s/.*/package work/' pkg/utils/utils.go > work/utils.go
133137
$(CONTROLLER_GEN) rbac:roleName=coil-egress-controller paths=./work output:stdout > $@
134-
# rm -rf work
138+
rm -rf work
135139

136140
COILD_DEPENDS = controllers/blockrequest_watcher.go \
137141
pkg/ipam/node.go \

‎v2/cmd/coil-egress-controller/sub/root.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"os"
77

88
v2 "github.com/cybozu-go/coil/v2"
9+
"github.com/cybozu-go/coil/v2/pkg/constants"
910
"github.com/spf13/cobra"
1011
"k8s.io/klog/v2"
1112
"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -18,6 +19,9 @@ var config struct {
1819
certDir string
1920
egressPort int32
2021
zapOpts zap.Options
22+
23+
disableCertRotation bool
24+
disableRestartOnCertRefresh bool
2125
}
2226

2327
var rootCmd = &cobra.Command{
@@ -47,6 +51,8 @@ func init() {
4751
pf.StringVar(&config.webhookAddr, "webhook-addr", ":9444", "bind address of admission webhook")
4852
pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
4953
pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress")
54+
pf.BoolVar(&config.disableCertRotation, "disable-cert-rotation", constants.DefaultDisableCertRotation, "disables webhook's certificate generation")
55+
pf.BoolVar(&config.disableRestartOnCertRefresh, "disable-restart-on-cert-refresh", constants.DefailtDisableRestartOnCertRefresh, "disables pod's restart on webhook certificate refresh")
5056

5157
goflags := flag.NewFlagSet("klog", flag.ExitOnError)
5258
klog.InitFlags(goflags)

‎v2/cmd/coil-egress-controller/sub/run.go‎

Lines changed: 41 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package sub
22

33
import (
4+
"context"
45
"fmt"
56
"net"
67
"os"
@@ -11,6 +12,7 @@ import (
1112
coilv2 "github.com/cybozu-go/coil/v2/api/v2"
1213
"github.com/cybozu-go/coil/v2/controllers"
1314
"github.com/cybozu-go/coil/v2/pkg/constants"
15+
"github.com/cybozu-go/coil/v2/pkg/utils"
1416
"k8s.io/apimachinery/pkg/runtime"
1517
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
1618
clientgoscheme "k8s.io/client-go/kubernetes/scheme"
@@ -78,9 +80,46 @@ func subMain() error {
7880
return err
7981
}
8082

81-
// register controllers
83+
var setupFinished chan struct{}
84+
85+
if !config.disableCertRotation {
86+
setupFinished, err = utils.SetupRotator(mgr, "egress", config.disableRestartOnCertRefresh)
87+
if err != nil {
88+
return fmt.Errorf("failed to setup Rotator: %w", err)
89+
}
90+
}
91+
92+
setupErr := make(chan error)
93+
94+
go func() {
95+
setupErr <- setupManager(mgr, setupFinished)
96+
close(setupErr)
97+
}()
8298

83-
ctx := ctrl.SetupSignalHandler()
99+
mgrCtx, cancel := context.WithCancel(ctrl.SetupSignalHandler())
100+
defer cancel()
101+
102+
mgrErr := make(chan error)
103+
go func() {
104+
setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
105+
if err := mgr.Start(mgrCtx); err != nil {
106+
mgrErr <- err
107+
}
108+
close(mgrErr)
109+
}()
110+
111+
if err := utils.WaitForExit(setupErr, mgrErr, cancel); err != nil {
112+
return err
113+
}
114+
115+
return nil
116+
}
117+
118+
func setupManager(mgr ctrl.Manager, setupFinished chan struct{}) error {
119+
// wait for certificates to be configured
120+
<-setupFinished
121+
122+
// register controllers
84123

85124
podNS := os.Getenv(constants.EnvPodNamespace)
86125
podName := os.Getenv(constants.EnvPodName)
@@ -108,13 +147,5 @@ func subMain() error {
108147
return err
109148
}
110149

111-
// start manager
112-
113-
setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
114-
if err := mgr.Start(ctx); err != nil {
115-
setupLog.Error(err, "problem running manager")
116-
return err
117-
}
118-
119150
return nil
120151
}

‎v2/cmd/coil-ipam-controller/sub/root.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"time"
88

99
v2 "github.com/cybozu-go/coil/v2"
10+
"github.com/cybozu-go/coil/v2/pkg/constants"
1011
"github.com/spf13/cobra"
1112
"k8s.io/klog/v2"
1213
"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -19,6 +20,9 @@ var config struct {
1920
certDir string
2021
gcInterval time.Duration
2122
zapOpts zap.Options
23+
24+
disableCertRotation bool
25+
disableRestartOnCertRefresh bool
2226
}
2327

2428
var rootCmd = &cobra.Command{
@@ -48,6 +52,8 @@ func init() {
4852
pf.StringVar(&config.webhookAddr, "webhook-addr", ":9443", "bind address of admission webhook")
4953
pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
5054
pf.DurationVar(&config.gcInterval, "gc-interval", 1*time.Hour, "garbage collection interval")
55+
pf.BoolVar(&config.disableCertRotation, "disable-cert-rotation", constants.DefaultDisableCertRotation, "disables webhook's certificate generation")
56+
pf.BoolVar(&config.disableRestartOnCertRefresh, "disable-restart-on-cert-refresh", constants.DefailtDisableRestartOnCertRefresh, "disables pod's restart on webhook certificate refresh")
5157

5258
goflags := flag.NewFlagSet("klog", flag.ExitOnError)
5359
klog.InitFlags(goflags)

‎v2/cmd/coil-ipam-controller/sub/run.go‎

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package sub
22

33
import (
4+
"context"
45
"fmt"
56
"net"
67
"strconv"
@@ -11,6 +12,7 @@ import (
1112
"github.com/cybozu-go/coil/v2/controllers"
1213
"github.com/cybozu-go/coil/v2/pkg/indexing"
1314
"github.com/cybozu-go/coil/v2/pkg/ipam"
15+
"github.com/cybozu-go/coil/v2/pkg/utils"
1416
"github.com/cybozu-go/coil/v2/runners"
1517
"k8s.io/apimachinery/pkg/runtime"
1618
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -78,6 +80,46 @@ func subMain() error {
7880
return err
7981
}
8082

83+
var setupFinished chan struct{}
84+
85+
if !config.disableCertRotation {
86+
setupFinished, err = utils.SetupRotator(mgr, "ipam", config.disableRestartOnCertRefresh)
87+
if err != nil {
88+
return fmt.Errorf("failed to setup Rotator: %w", err)
89+
}
90+
}
91+
92+
ctx := ctrl.SetupSignalHandler()
93+
94+
setupErr := make(chan error)
95+
96+
go func() {
97+
setupErr <- setupManager(ctx, mgr, setupFinished)
98+
close(setupErr)
99+
}()
100+
101+
mgrCtx, cancel := context.WithCancel(ctx)
102+
103+
mgrErr := make(chan error)
104+
go func() {
105+
setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
106+
if err := mgr.Start(mgrCtx); err != nil {
107+
mgrErr <- err
108+
}
109+
close(mgrErr)
110+
}()
111+
112+
if err := utils.WaitForExit(setupErr, mgrErr, cancel); err != nil {
113+
return err
114+
}
115+
116+
return nil
117+
}
118+
119+
func setupManager(ctx context.Context, mgr ctrl.Manager, setupFinished chan struct{}) error {
120+
// wait for certificates to be configured
121+
<-setupFinished
122+
81123
// register controllers
82124

83125
pm := ipam.NewPoolManager(mgr.GetClient(), mgr.GetAPIReader(), ctrl.Log.WithName("pool-manager"), scheme)
@@ -90,7 +132,6 @@ func subMain() error {
90132
return err
91133
}
92134

93-
ctx := ctrl.SetupSignalHandler()
94135
if err := indexing.SetupIndexForAddressBlock(ctx, mgr); err != nil {
95136
return err
96137
}
@@ -117,11 +158,5 @@ func subMain() error {
117158
return err
118159
}
119160

120-
setupLog.Info(fmt.Sprintf("starting manager (version: %s)", v2.Version()))
121-
if err := mgr.Start(ctx); err != nil {
122-
setupLog.Error(err, "problem running manager")
123-
return err
124-
}
125-
126161
return nil
127162
}

‎v2/config/default/egress/v4/kustomization.yaml‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,19 @@ resources:
44
- ../../../pod/egress/v4
55
- ../../../webhook/egress
66

7-
patchesStrategicMerge:
8-
- ../webhook_manifests_patch.yaml
7+
# [CERTS] Following lines should be uncommented if pre-generated certs are used.
8+
# patchesStrategicMerge:
9+
# - ../webhook_manifests_patch.yaml
910

1011
generatorOptions:
1112
disableNameSuffixHash: true
1213

1314
secretGenerator:
1415
# [EGRESS] Following lines be uncommented to enable Egress NAT features.
1516
- name: coilv2-egress-webhook-server-cert
16-
files:
17-
- ca.crt=../../cert.pem
18-
- tls.crt=../../egress-cert.pem
19-
- tls.key=../../egress-key.pem
20-
type: "kubernetes.io/tls"
17+
# [CERTS] Following lines should be uncommented if pre-generated certs are used.
18+
# files:
19+
# - ca.crt=../../cert.pem
20+
# - tls.crt=../../egress-cert.pem
21+
# - tls.key=../../egress-key.pem
22+
# type: "kubernetes.io/tls"

‎v2/config/default/egress/v6/kustomization.yaml‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,19 @@ resources:
44
- ../../../pod/egress/v6
55
- ../../../webhook/egress
66

7-
patchesStrategicMerge:
8-
- ../webhook_manifests_patch.yaml
7+
# [CERTS] Following lines should be uncommented if pre-generated certs are used.
8+
# patchesStrategicMerge:
9+
# - ../webhook_manifests_patch.yaml
910

1011
generatorOptions:
1112
disableNameSuffixHash: true
1213

1314
secretGenerator:
1415
# [EGRESS] Following lines be uncommented to enable Egress NAT features.
1516
- name: coilv2-egress-webhook-server-cert
16-
files:
17-
- ca.crt=../../cert.pem
18-
- tls.crt=../../egress-cert.pem
19-
- tls.key=../../egress-key.pem
20-
type: "kubernetes.io/tls"
17+
# [CERTS] Following lines should be uncommented if pre-generated certs are used.
18+
# files:
19+
# - ca.crt=../../cert.pem
20+
# - tls.crt=../../egress-cert.pem
21+
# - tls.key=../../egress-key.pem
22+
# type: "kubernetes.io/tls"

0 commit comments

Comments
 (0)