Skip to content

Commit 255a03e

Browse files
jimmidysonjimmidyson
authored
feat: Allow configuration of kube-proxy mode on cluster creation (#1163)
This commit allows users to configure kube-proxy mode when creating clusters. This is enforced via CEL to prevent users from moving between kube-proxy and non-kube-proxy deployments. Supports `iptables` (default), and `nftables` modes. Skipping `kube-proxy` deployment and management by CAPI is enabled via setting the upstream `controlplane.cluster.x-k8s.io/skip-kube-proxy` annotation on the `Cluster` `spec.topology.controlPlane.metadata.annotation`. This will configure `kubeadm` to skip the `addon/kube-proxy` phase. Follow up work will allow Cilium configuration to enable their kube-proxy replacement (already possible via custom Helm values) and migration from kube-proxy to kube-proxy replacement for existing clusters. Requires #1162.
1 parent 73edfd8 commit 255a03e

File tree

13 files changed

+1061
-5
lines changed

13 files changed

+1061
-5
lines changed

‎api/v1alpha1/clusterconfig_types.go

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,10 @@ type GenericClusterConfigSpec struct {
212212

213213
// +kubebuilder:validation:Optional
214214
DNS *DNS `json:"dns,omitempty"`
215+
216+
// KubeProxy defines the configuration for kube-proxy.
217+
// +kubebuilder:validation:Optional
218+
KubeProxy *KubeProxy `json:"kubeProxy,omitempty"`
215219
}
216220

217221
type Image struct {
@@ -328,6 +332,27 @@ type CoreDNS struct {
328332
Image *Image `json:"image,omitempty"`
329333
}
330334

335+
type KubeProxyMode string
336+
337+
const (
338+
// KubeProxyModeIPTables indicates that kube-proxy should be installed in iptables
339+
// mode.
340+
KubeProxyModeIPTables KubeProxyMode = "iptables"
341+
// KubeProxyModeNFTables indicates that kube-proxy should be installed in nftables
342+
// mode.
343+
KubeProxyModeNFTables KubeProxyMode = "nftables"
344+
)
345+
346+
type KubeProxy struct {
347+
// Mode specifies the mode for kube-proxy:
348+
// - iptables means that kube-proxy is installed in iptables mode.
349+
// - nftables means that kube-proxy is installed in nftables mode.
350+
// +kubebuilder:validation:Optional
351+
// +kubebuilder:validation:Enum=iptables;nftables
352+
// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value cannot be changed after cluster creation"
353+
Mode KubeProxyMode `json:"mode,omitempty"`
354+
}
355+
331356
//nolint:gochecknoinits // Idiomatic to use init functions to register APIs with scheme.
332357
func init() {
333358
objectTypes = append(objectTypes,

‎api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,22 @@ spec:
560560
- url
561561
type: object
562562
type: array
563+
kubeProxy:
564+
description: KubeProxy defines the configuration for kube-proxy.
565+
properties:
566+
mode:
567+
description: |-
568+
Mode specifies the mode for kube-proxy:
569+
- iptables means that kube-proxy is installed in iptables mode.
570+
- nftables means that kube-proxy is installed in nftables mode.
571+
enum:
572+
- iptables
573+
- nftables
574+
type: string
575+
x-kubernetes-validations:
576+
- message: Value cannot be changed after cluster creation
577+
rule: self == oldSelf
578+
type: object
563579
kubernetesImageRepository:
564580
description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
565581
pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$

‎api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -497,6 +497,22 @@ spec:
497497
- url
498498
type: object
499499
type: array
500+
kubeProxy:
501+
description: KubeProxy defines the configuration for kube-proxy.
502+
properties:
503+
mode:
504+
description: |-
505+
Mode specifies the mode for kube-proxy:
506+
- iptables means that kube-proxy is installed in iptables mode.
507+
- nftables means that kube-proxy is installed in nftables mode.
508+
enum:
509+
- iptables
510+
- nftables
511+
type: string
512+
x-kubernetes-validations:
513+
- message: Value cannot be changed after cluster creation
514+
rule: self == oldSelf
515+
type: object
500516
kubernetesImageRepository:
501517
description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
502518
pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$

‎api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,22 @@ spec:
175175
- url
176176
type: object
177177
type: array
178+
kubeProxy:
179+
description: KubeProxy defines the configuration for kube-proxy.
180+
properties:
181+
mode:
182+
description: |-
183+
Mode specifies the mode for kube-proxy:
184+
- iptables means that kube-proxy is installed in iptables mode.
185+
- nftables means that kube-proxy is installed in nftables mode.
186+
enum:
187+
- iptables
188+
- nftables
189+
type: string
190+
x-kubernetes-validations:
191+
- message: Value cannot be changed after cluster creation
192+
rule: self == oldSelf
193+
type: object
178194
kubernetesImageRepository:
179195
description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
180196
pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$

‎api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -676,6 +676,22 @@ spec:
676676
- url
677677
type: object
678678
type: array
679+
kubeProxy:
680+
description: KubeProxy defines the configuration for kube-proxy.
681+
properties:
682+
mode:
683+
description: |-
684+
Mode specifies the mode for kube-proxy:
685+
- iptables means that kube-proxy is installed in iptables mode.
686+
- nftables means that kube-proxy is installed in nftables mode.
687+
enum:
688+
- iptables
689+
- nftables
690+
type: string
691+
x-kubernetes-validations:
692+
- message: Value cannot be changed after cluster creation
693+
rule: self == oldSelf
694+
type: object
679695
kubernetesImageRepository:
680696
description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
681697
pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$

‎api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 20 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/content/customization/generic/kube-proxy-mode.md

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
+++
2+
title = "kube-proxy mode"
3+
+++
4+
5+
This customization allows configuration of the `kube-proxy` proxy mode. Currently, only `iptables` and `nftables`
6+
modes are supported. By default, `kube-proxy` is enabled in `iptables` mode by `kubeadm`.
7+
8+
## Examples
9+
10+
### Enabling nftables kube-proxy mode
11+
12+
Enabling `nftables` is done via the following configuration:
13+
14+
```yaml
15+
apiVersion: cluster.x-k8s.io/v1beta1
16+
kind: Cluster
17+
metadata:
18+
name: <NAME>
19+
spec:
20+
topology:
21+
variables:
22+
- name: clusterConfig
23+
value:
24+
kubeProxy:
25+
mode: nftables
26+
```
27+
28+
Applying this configuration will result in the following configuration being applied to create a
29+
`KubeProxyConfiguration` and append it to the kubeadm configuration that is used when `kubeadm init`
30+
is executed:
31+
32+
- `KubeadmControlPlaneTemplate`:
33+
34+
- ```yaml
35+
spec:
36+
template:
37+
spec:
38+
kubeadmConfigSpec:
39+
files:
40+
- path: "/etc/kubernetes/kubeproxy-config.yaml"
41+
owner: "root:root"
42+
permissions: "0644"
43+
content: |-
44+
---
45+
apiVersion: kubeproxy.config.k8s.io/v1alpha1
46+
kind: KubeProxyConfiguration
47+
mode: nftables
48+
preKubeadmCommands:
49+
- /bin/sh -ec 'cat /etc/kubernetes/kubeproxy-config.yaml >> /run/kubeadm/kubeadm.yaml'
50+
```
51+
52+
### Skipping kube-proxy installation
53+
54+
To disable the deployment and upgrade of `kube-proxy`, specify the following configuration:
55+
56+
```yaml
57+
apiVersion: cluster.x-k8s.io/v1beta1
58+
kind: Cluster
59+
metadata:
60+
name: <NAME>
61+
spec:
62+
topology:
63+
controlPlane:
64+
metadata:
65+
annotations:
66+
controlplane.cluster.x-k8s.io/skip-kube-proxy: ""
67+
```
68+
69+
Applying this configuration will result in the following configuration being applied:
70+
71+
- `KubeadmControlPlaneTemplate`:
72+
73+
- ```yaml
74+
spec:
75+
template:
76+
spec:
77+
kubeadmConfigSpec:
78+
initConfiguration:
79+
skipPhases:
80+
- addon/kube-proxy
81+
```

‎pkg/handlers/generic/mutation/extraapiservercertsans/inject_test.go

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -175,12 +175,10 @@ var _ = Describe("Generate Extra API server certificate patches", func() {
175175
utilruntime.Must(clientgoscheme.AddToScheme(clientScheme))
176176
utilruntime.Must(clusterv1.AddToScheme(clientScheme))
177177
cl, err := helpers.TestEnv.GetK8sClientWithScheme(clientScheme)
178-
gomega.Expect(err).To(gomega.BeNil())
179-
err = cl.Create(context.Background(), &tt.cluster)
180-
gomega.Expect(err).To(gomega.BeNil())
178+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
179+
gomega.Expect(cl.Create(context.Background(), &tt.cluster)).To(gomega.Succeed())
180+
DeferCleanup(cl.Delete, context.Background(), &tt.cluster)
181181
capitest.AssertGeneratePatches(GinkgoT(), patchGenerator, &tt.patchTest)
182-
err = cl.Delete(context.Background(), &tt.cluster)
183-
gomega.Expect(err).To(gomega.BeNil())
184182
})
185183
}
186184
})

‎pkg/handlers/generic/mutation/handlers.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/extraapiservercertsans"
2121
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/httpproxy"
2222
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/imageregistries/credentials"
23+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubeproxymode"
2324
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubernetesimagerepository"
2425
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors"
2526
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/noderegistration"
@@ -44,6 +45,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
4445
containerdunprivilegedports.NewPatch(),
4546
encryptionatrest.NewPatch(mgr.GetClient(), encryptionatrest.RandomTokenGenerator),
4647
autorenewcerts.NewPatch(),
48+
kubeproxymode.NewPatch(),
4749

4850
// Some patches may have changed containerd configuration.
4951
// We write the configuration changes to disk, and must run a command

‎pkg/handlers/generic/mutation/kubeproxymode/embedded/kubeproxyconfig.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
---
2+
apiVersion: kubeproxy.config.k8s.io/v1alpha1
3+
kind: KubeProxyConfiguration
4+
mode: {{ .Mode }}

0 commit comments

Comments
 (0)