Skip to content

Commit 2007a42

Browse files
feat(ntp): Configure NTP for clusters (#1185)
* Add API for configuring NTP servers * Add mutation handlers for injecting NTP servers **How has this been tested?** * Create a cluster with custom NTP configuration ``` apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster metadata: annotations: caren.nutanix.com/cluster-uuid: 0197a351-97fa-7b0c-88a7-aa83fb309884 labels: cluster.x-k8s.io/provider: nutanix konvoy.d2iq.io/cluster-name: nkp-sid konvoy.d2iq.io/provider: nutanix name: nkp-sid namespace: default spec: clusterNetwork: pods: cidrBlocks: - 192.168.0.0/16 services: cidrBlocks: - 10.96.0.0/12 controlPlaneEndpoint: host: "" port: 0 topology: class: nutanix-quick-start controlPlane: metadata: {} replicas: 3 variables: - name: clusterConfig value: ... ntp: servers: - time.google.com ``` * Observe the kubeadmconfigs for machines have NTP Server set correctly ``` $ k get kubeadmconfigs -o yaml | grep ntp -A 3 ntp: enabled: true servers: - time.google.com -- ntp: enabled: true servers: - time.google.com -- ntp: enabled: true servers: - time.google.com -- ntp: enabled: true servers: - time.google.com -- ntp: enabled: true servers: - time.google.com ``` * Update the ntp config to point to a different NTP server and observe a machine rollout with new kubeadmconfigs set to the new NTP server ``` $ k get kubeadmconfigs -o yaml | grep ntp -A 3 ntp: enabled: true servers: - time.aws.com -- ntp: enabled: true servers: - time.aws.com -- ntp: enabled: true servers: - time.aws.com -- ntp: enabled: true servers: - time.aws.com -- ntp: enabled: true servers: - time.aws.com ``` --------- Co-authored-by: Daniel Lipovetsky <daniel.lipovetsky@nutanix.com>
1 parent 82bd135 commit 2007a42

11 files changed

+510
-0
lines changed

api/v1alpha1/clusterconfig_types.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,10 @@ type GenericClusterConfigSpec struct {
216216
// KubeProxy defines the configuration for kube-proxy.
217217
// +kubebuilder:validation:Optional
218218
KubeProxy *KubeProxy `json:"kubeProxy,omitempty"`
219+
220+
// NTP defines the NTP configuration for the cluster.
221+
// +kubebuilder:validation:Optional
222+
NTP *NTP `json:"ntp,omitempty"`
219223
}
220224

221225
type Image struct {
@@ -353,6 +357,14 @@ type KubeProxy struct {
353357
Mode KubeProxyMode `json:"mode,omitempty"`
354358
}
355359

360+
// NTP defines the NTP configuration for the cluster.
361+
type NTP struct {
362+
// Servers is a list of NTP servers to use for time synchronization.
363+
// +kubebuilder:validation:Required
364+
// +kubebuilder:validation:MinItems=1
365+
Servers []string `json:"servers"`
366+
}
367+
356368
//nolint:gochecknoinits // Idiomatic to use init functions to register APIs with scheme.
357369
func init() {
358370
objectTypes = append(objectTypes,

api/v1alpha1/crds/caren.nutanix.com_awsclusterconfigs.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -580,6 +580,18 @@ spec:
580580
description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
581581
pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
582582
type: string
583+
ntp:
584+
description: NTP defines the NTP configuration for the cluster.
585+
properties:
586+
servers:
587+
description: Servers is a list of NTP servers to use for time synchronization.
588+
items:
589+
type: string
590+
minItems: 1
591+
type: array
592+
required:
593+
- servers
594+
type: object
583595
proxy:
584596
description: HTTPProxy required for providing proxy configuration.
585597
properties:

api/v1alpha1/crds/caren.nutanix.com_dockerclusterconfigs.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -517,6 +517,18 @@ spec:
517517
description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
518518
pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
519519
type: string
520+
ntp:
521+
description: NTP defines the NTP configuration for the cluster.
522+
properties:
523+
servers:
524+
description: Servers is a list of NTP servers to use for time synchronization.
525+
items:
526+
type: string
527+
minItems: 1
528+
type: array
529+
required:
530+
- servers
531+
type: object
520532
proxy:
521533
description: HTTPProxy required for providing proxy configuration.
522534
properties:

api/v1alpha1/crds/caren.nutanix.com_genericclusterconfigs.yaml

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,19 @@ spec:
195195
description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
196196
pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
197197
type: string
198+
ntp:
199+
description: NTP defines the NTP configuration for the cluster.
200+
properties:
201+
servers:
202+
description: Servers is a list of NTP servers to use for time
203+
synchronization.
204+
items:
205+
type: string
206+
minItems: 1
207+
type: array
208+
required:
209+
- servers
210+
type: object
198211
proxy:
199212
description: HTTPProxy required for providing proxy configuration.
200213
properties:

api/v1alpha1/crds/caren.nutanix.com_nutanixclusterconfigs.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -696,6 +696,18 @@ spec:
696696
description: Sets the Kubernetes image repository used for the KubeadmControlPlane.
697697
pattern: ^((?:[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*|\[(?:[a-fA-F0-9:]+)\])(:[0-9]+)?/)?[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*(/[a-z0-9]+((?:[._]|__|[-]+)[a-z0-9]+)*)*$
698698
type: string
699+
ntp:
700+
description: NTP defines the NTP configuration for the cluster.
701+
properties:
702+
servers:
703+
description: Servers is a list of NTP servers to use for time synchronization.
704+
items:
705+
type: string
706+
minItems: 1
707+
type: array
708+
required:
709+
- servers
710+
type: object
699711
nutanix:
700712
description: NutanixSpec defines the desired state of NutanixCluster.
701713
properties:

api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 25 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
+++
2+
title = "Network Time Protocol (NTP)"
3+
+++
4+
5+
You can configure cluster nodes (control plane, and workers) to update their system clock from a specific set of NTP servers.
6+
7+
Keep in mind that each node's operating system is almost certainly configured with a default set of NTP servers.
8+
In some cases, the node must use different NTP servers. For example, if the node cannot reach the default servers.
9+
10+
## Example
11+
12+
```yaml
13+
apiVersion: cluster.x-k8s.io/v1beta1
14+
kind: Cluster
15+
metadata:
16+
name: <NAME>
17+
spec:
18+
topology:
19+
variables:
20+
- name: clusterConfig
21+
value:
22+
ntp
23+
servers:
24+
- time1.example.com
25+
- time2.example.com
26+
```
27+
28+
Applying this configuration will result in the following values being set
29+
30+
for the control plane, on the `KubeadmControlPlaneTemplate`
31+
32+
```yaml
33+
spec:
34+
kubeadmConfigSpec:
35+
ntp:
36+
enabled: true
37+
servers:
38+
- time1.example.com
39+
- time2.example.com
40+
```
41+
42+
and for every worker, on its respective `KubeadmConfigTemplate`
43+
44+
```yaml
45+
spec:
46+
ntp:
47+
enabled: true
48+
servers:
49+
- time1.example.com
50+
- time2.example.com
51+
```

pkg/handlers/generic/mutation/handlers.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import (
2424
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/kubernetesimagerepository"
2525
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/mirrors"
2626
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/noderegistration"
27+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/ntp"
2728
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/taints"
2829
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/pkg/handlers/generic/mutation/users"
2930
)
@@ -46,6 +47,7 @@ func MetaMutators(mgr manager.Manager) []mutation.MetaMutator {
4647
encryptionatrest.NewPatch(mgr.GetClient(), encryptionatrest.RandomTokenGenerator),
4748
autorenewcerts.NewPatch(),
4849
kubeproxymode.NewPatch(),
50+
ntp.NewPatch(),
4951

5052
// Some patches may have changed containerd configuration.
5153
// We write the configuration changes to disk, and must run a command
Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Copyright 2023 Nutanix. All rights reserved.
2+
// SPDX-License-Identifier: Apache-2.0
3+
4+
package ntp
5+
6+
import (
7+
"context"
8+
9+
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
10+
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
11+
"k8s.io/utils/ptr"
12+
bootstrapv1 "sigs.k8s.io/cluster-api/bootstrap/kubeadm/api/v1beta1"
13+
controlplanev1 "sigs.k8s.io/cluster-api/controlplane/kubeadm/api/v1beta1"
14+
runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
15+
ctrl "sigs.k8s.io/controller-runtime"
16+
"sigs.k8s.io/controller-runtime/pkg/client"
17+
18+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/api/v1alpha1"
19+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/handlers/mutation"
20+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches"
21+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/patches/selectors"
22+
"github.com/nutanix-cloud-native/cluster-api-runtime-extensions-nutanix/common/pkg/capi/clustertopology/variables"
23+
)
24+
25+
const (
26+
// VariableName is the external patch variable name.
27+
VariableName = "ntp"
28+
)
29+
30+
type ntpPatchHandler struct {
31+
variableName string
32+
variableFieldPath []string
33+
}
34+
35+
func NewPatch() *ntpPatchHandler {
36+
return newNTPPatchHandler(v1alpha1.ClusterConfigVariableName, VariableName)
37+
}
38+
39+
func newNTPPatchHandler(
40+
variableName string,
41+
variableFieldPath ...string,
42+
) *ntpPatchHandler {
43+
return &ntpPatchHandler{
44+
variableName: variableName,
45+
variableFieldPath: variableFieldPath,
46+
}
47+
}
48+
49+
func (h *ntpPatchHandler) Mutate(
50+
ctx context.Context,
51+
obj *unstructured.Unstructured,
52+
vars map[string]apiextensionsv1.JSON,
53+
holderRef runtimehooksv1.HolderReference,
54+
_ client.ObjectKey,
55+
_ mutation.ClusterGetter,
56+
) error {
57+
log := ctrl.LoggerFrom(ctx).WithValues(
58+
"holderRef", holderRef,
59+
)
60+
61+
ntp, err := variables.Get[*v1alpha1.NTP](
62+
vars,
63+
h.variableName,
64+
h.variableFieldPath...,
65+
)
66+
if err != nil {
67+
if variables.IsNotFoundError(err) {
68+
log.V(5).Info("NTP variable not defined")
69+
return nil
70+
}
71+
return err
72+
}
73+
74+
log = log.WithValues(
75+
"variableName",
76+
h.variableName,
77+
"variableFieldPath",
78+
h.variableFieldPath,
79+
"variableValue",
80+
ntp,
81+
)
82+
83+
if ntp == nil || len(ntp.Servers) == 0 {
84+
log.V(5).Info("NTP servers not specified, skipping mutation")
85+
return nil
86+
}
87+
88+
if err := patches.MutateIfApplicable(
89+
obj, vars, &holderRef, selectors.ControlPlane(), log,
90+
func(obj *controlplanev1.KubeadmControlPlaneTemplate) error {
91+
log.WithValues(
92+
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
93+
"patchedObjectName", client.ObjectKeyFromObject(obj),
94+
).Info("setting NTP configuration in control plane kubeadm config spec")
95+
96+
obj.Spec.Template.Spec.KubeadmConfigSpec.NTP = &bootstrapv1.NTP{
97+
Enabled: ptr.To(true),
98+
Servers: ntp.Servers,
99+
}
100+
101+
return nil
102+
}); err != nil {
103+
return err
104+
}
105+
106+
if err := patches.MutateIfApplicable(
107+
obj, vars, &holderRef, selectors.WorkersKubeadmConfigTemplateSelector(), log,
108+
func(obj *bootstrapv1.KubeadmConfigTemplate) error {
109+
log.WithValues(
110+
"patchedObjectKind", obj.GetObjectKind().GroupVersionKind().String(),
111+
"patchedObjectName", client.ObjectKeyFromObject(obj),
112+
).Info("setting NTP configuration in worker kubeadm config spec")
113+
114+
obj.Spec.Template.Spec.NTP = &bootstrapv1.NTP{
115+
Enabled: ptr.To(true),
116+
Servers: ntp.Servers,
117+
}
118+
119+
return nil
120+
}); err != nil {
121+
return err
122+
}
123+
124+
return nil
125+
}

0 commit comments

Comments
 (0)