Skip to content

Commit f2fab91

Browse files
jimmidysonjimmidyson
authored
feat: Add scale from zero cluster-autoscaler support (#1227)
Added documentation for how to use with providers that do not natively support Cluster Autoscaler scale-from-zero. Tested with CAPA (native support) and CAPD (via annotations) and works nicely. **What problem does this PR solve?**: **Which issue(s) this PR fixes**: Fixes # **How Has This Been Tested?**: <!-- Please describe the tests that you ran to verify your changes. Provide output from the tests and any manual steps needed to replicate the tests. --> **Special notes for your reviewer**: <!-- Use this to provide any additional information to the reviewers. This may include: - Best way to review the PR. - Where the author wants the most review attention on. - etc. -->
1 parent 5f8c4d7 commit f2fab91

File tree

7 files changed

+131
-10
lines changed

7 files changed

+131
-10
lines changed

‎charts/cluster-api-runtime-extensions-nutanix/addons/cluster-autoscaler/values-template.yaml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,3 +33,35 @@ extraVolumeSecrets:
3333
rbac:
3434
# Create a Role instead of a ClusterRoles to update cluster-api objects
3535
clusterScoped: false
36+
{{- $capiProvider := index .Cluster.Labels "cluster.x-k8s.io/provider" }}
37+
{{- if eq $capiProvider "aws" }}
38+
additionalRules:
39+
- apiGroups:
40+
- infrastructure.cluster.x-k8s.io
41+
resources:
42+
- awsmachinetemplates
43+
verbs:
44+
- get
45+
- list
46+
- watch
47+
{{- else if eq $capiProvider "docker" }}
48+
additionalRules:
49+
- apiGroups:
50+
- infrastructure.cluster.x-k8s.io
51+
resources:
52+
- dockermachinetemplates
53+
verbs:
54+
- get
55+
- list
56+
- watch
57+
{{- else if eq $capiProvider "nutanix" }}
58+
additionalRules:
59+
- apiGroups:
60+
- infrastructure.cluster.x-k8s.io
61+
resources:
62+
- nutanixmachinetemplates
63+
verbs:
64+
- get
65+
- list
66+
- watch
67+
{{- end }}

‎charts/cluster-api-runtime-extensions-nutanix/templates/cluster-autoscaler/manifests/cluster-autoscaler-configmap.yaml

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ data:
1515
app.kubernetes.io/instance: ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
1616
app.kubernetes.io/managed-by: Helm
1717
app.kubernetes.io/name: clusterapi-cluster-autoscaler
18-
helm.sh/chart: cluster-autoscaler-9.46.6
18+
helm.sh/chart: cluster-autoscaler-9.48.0
1919
name: cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
2020
namespace: {{ `{{ .Cluster.Namespace }}` }}
2121
spec:
@@ -33,7 +33,7 @@ data:
3333
app.kubernetes.io/instance: ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
3434
app.kubernetes.io/managed-by: Helm
3535
app.kubernetes.io/name: clusterapi-cluster-autoscaler
36-
helm.sh/chart: cluster-autoscaler-9.46.6
36+
helm.sh/chart: cluster-autoscaler-9.48.0
3737
name: cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
3838
namespace: {{ `{{ .Cluster.Namespace }}` }}
3939
---
@@ -44,7 +44,7 @@ data:
4444
app.kubernetes.io/instance: ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
4545
app.kubernetes.io/managed-by: Helm
4646
app.kubernetes.io/name: clusterapi-cluster-autoscaler
47-
helm.sh/chart: cluster-autoscaler-9.46.6
47+
helm.sh/chart: cluster-autoscaler-9.48.0
4848
name: cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
4949
namespace: {{ `{{ .Cluster.Namespace }}` }}
5050
rules:
@@ -108,7 +108,7 @@ data:
108108
app.kubernetes.io/instance: ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
109109
app.kubernetes.io/managed-by: Helm
110110
app.kubernetes.io/name: clusterapi-cluster-autoscaler
111-
helm.sh/chart: cluster-autoscaler-9.46.6
111+
helm.sh/chart: cluster-autoscaler-9.48.0
112112
name: cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
113113
namespace: {{ `{{ .Cluster.Namespace }}` }}
114114
roleRef:
@@ -127,7 +127,7 @@ data:
127127
app.kubernetes.io/instance: ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
128128
app.kubernetes.io/managed-by: Helm
129129
app.kubernetes.io/name: clusterapi-cluster-autoscaler
130-
helm.sh/chart: cluster-autoscaler-9.46.6
130+
helm.sh/chart: cluster-autoscaler-9.48.0
131131
name: cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
132132
namespace: {{ `{{ .Cluster.Namespace }}` }}
133133
spec:
@@ -148,7 +148,7 @@ data:
148148
app.kubernetes.io/instance: ca-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
149149
app.kubernetes.io/managed-by: Helm
150150
app.kubernetes.io/name: clusterapi-cluster-autoscaler
151-
helm.sh/chart: cluster-autoscaler-9.46.6
151+
helm.sh/chart: cluster-autoscaler-9.48.0
152152
name: cluster-autoscaler-{{ `{{ index .Cluster.Annotations "caren.nutanix.com/cluster-uuid" }}` }}
153153
namespace: {{ `{{ .Cluster.Namespace }}` }}
154154
spec:
@@ -185,7 +185,7 @@ data:
185185
valueFrom:
186186
fieldRef:
187187
fieldPath: spec.serviceAccountName
188-
image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.32.0
188+
image: registry.k8s.io/autoscaling/cluster-autoscaler:v1.33.0
189189
imagePullPolicy: IfNotPresent
190190
livenessProbe:
191191
httpGet:

‎charts/cluster-api-runtime-extensions-nutanix/templates/helm-config.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ data:
2121
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://helm.cilium.io/{{ end }}'
2222
cluster-autoscaler: |
2323
ChartName: cluster-autoscaler
24-
ChartVersion: 9.46.6
24+
ChartVersion: 9.48.0
2525
RepositoryURL: '{{ if .Values.helmRepository.enabled }}oci://helm-repository.{{ .Release.Namespace }}.svc/charts{{ else }}https://kubernetes.github.io/autoscaler{{ end }}'
2626
cncf-distribution-registry: |
2727
ChartName: docker-registry

‎docs/content/addons/cluster-autoscaler.md

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,92 @@ spec:
5050

5151
To deploy the addon via `ClusterResourceSet` replace the value of `strategy` with `ClusterResourceSet`.
5252

53+
## Scale from zero
54+
55+
{{% alert title="Required Cluster labels" color=warning %}}
56+
CAREN support for scale from zero currently relies on specific labels on the `Cluster` resource in order for Cluster
57+
Autoscaler RBAC to be correctly configured. Ensure that your `Clusters` have the appropriate `cluster.x-k8s.io/provider`
58+
label as follows:
59+
60+
- CAPA: `cluster.x-k8s.io/provider: aws`
61+
- CAPX: `cluster.x-k8s.io/provider: nutanix`
62+
- CAPD: `cluster.x-k8s.io/provider: docker`
63+
{{% /alert %}}
64+
65+
CAREN deploys Cluster Autoscaler with appropriate permissions to enable scaling nodepools from zero. However, CAPI
66+
providers must implement functionality as described in the [autoscaling from zero proposal][Scale from zero status
67+
updates] in order for scaling from zero to be possible.
68+
69+
For those providers that have not implemented this (e.g. Docker, Nutanix), scaling from zero is still possible by
70+
providing annotations on the `MachineDeployments` to allow Cluster Autoscaler to make appropriate scaling decisions.
71+
The following example shows the required annotations to add to:
72+
73+
```yaml
74+
apiVersion: cluster.x-k8s.io/v1beta1
75+
kind: Cluster
76+
metadata:
77+
name: <NAME>
78+
labels:
79+
cluster.x-k8s.io/provider: <PROVIDER>
80+
spec:
81+
topology:
82+
variables:
83+
- name: clusterConfig
84+
value:
85+
addons:
86+
clusterAutoscaler:
87+
strategy: HelmAddon
88+
workers:
89+
machineDeployments:
90+
- class: default-worker
91+
metadata:
92+
annotations:
93+
# Set the following annotations to configure the Cluster Autoscaler
94+
# The initial MachineDeployment will have 0 Machines
95+
cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "3"
96+
cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0"
97+
capacity.cluster-autoscaler.kubernetes.io/cpu: "8"
98+
capacity.cluster-autoscaler.kubernetes.io/memory: "8112564Ki"
99+
name: scale-from-zero-example
100+
# Do not set the replicas field, otherwise the topology controller will revert back the autoscaler's changes
101+
```
102+
103+
If the nodepool is labelled and/or tainted, additional annotations are required in order for Cluster Autoscaler to take
104+
these labels and taints into account to scale nodepools that have node affinity and/or tolerations configured:
105+
106+
```yaml
107+
apiVersion: cluster.x-k8s.io/v1beta1
108+
kind: Cluster
109+
metadata:
110+
name: <NAME>
111+
labels:
112+
cluster.x-k8s.io/provider: <PROVIDER>
113+
spec:
114+
topology:
115+
variables:
116+
- name: clusterConfig
117+
value:
118+
addons:
119+
clusterAutoscaler:
120+
strategy: HelmAddon
121+
workers:
122+
machineDeployments:
123+
- class: default-worker
124+
metadata:
125+
annotations:
126+
# Set the following annotations to configure the Cluster Autoscaler
127+
# The initial MachineDeployment will have 0 Machines
128+
cluster.x-k8s.io/cluster-api-autoscaler-node-group-max-size: "3"
129+
cluster.x-k8s.io/cluster-api-autoscaler-node-group-min-size: "0"
130+
capacity.cluster-autoscaler.kubernetes.io/cpu: "8"
131+
capacity.cluster-autoscaler.kubernetes.io/memory: "8112564Ki"
132+
capacity.cluster-autoscaler.kubernetes.io/labels: "node-restriction.kubernetes.io/my-app="
133+
capacity.cluster-autoscaler.kubernetes.io/taints: "mytaint=tainted:NoSchedule"
134+
name: scale-from-zero-example
135+
# Do not set the replicas field, otherwise the topology controller will revert back the autoscaler's changes
136+
```
137+
53138
[Cluster Autoscaler]: https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/clusterapi
54139
[Cluster API Add-on Provider for Helm]: https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm
55140
[Pivot]: https://main.cluster-api.sigs.k8s.io/clusterctl/commands/move#pivot
141+
[Scale from zero status updates]: https://github.com/kubernetes-sigs/cluster-api/blob/v1.10.4/docs/proposals/20210310-opt-in-autoscaling-from-zero.md#infrastructure-machine-template-status-updates

‎hack/addons/helm-chart-bundler/repos.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ repositories:
2525
repoURL: https://kubernetes.github.io/autoscaler
2626
charts:
2727
cluster-autoscaler:
28-
- 9.46.6
28+
- 9.48.0
2929
cosi:
3030
repoURL: https://mesosphere.github.io/charts/stable/
3131
charts:

‎hack/addons/update-cluster-autoscaler.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ Cluster:
3333
Namespace: tmpl-clusternamespace-tmpl
3434
Annotations:
3535
caren.nutanix.com/cluster-uuid: tmpl-clusteruuid-tmpl
36+
Labels:
37+
cluster.x-k8s.io/provider: tmpl-capiprovider-tmpl
3638
EOF
3739
gomplate -f "${GIT_REPO_ROOT}/charts/cluster-api-runtime-extensions-nutanix/addons/cluster-autoscaler/values-template.yaml" \
3840
--context .="${ASSETS_DIR}/gomplate-context.yaml" \
@@ -42,6 +44,7 @@ kustomize build --enable-helm "${ASSETS_DIR}" >"${ASSETS_DIR}/${FILE_NAME}"
4244

4345
sed -i -e "s/\([a-z-]*\)tmpl-clustername\([^-]*\)-tmpl\([a-z-]*\)/\1{{ \`{{ .Cluster.Name\2 }}\` }}\3/g" \
4446
-e "s/\([a-z-]*\)tmpl-clusteruuid-tmpl\([a-z-]*\)/\1{{ \`{{ index .Cluster.Annotations \"caren.nutanix.com\/cluster-uuid\" }}\` }}\2/g" \
47+
-e "s/\([a-z-]*\)tmpl-capiprovider-tmpl\([a-z-]*\)/\1{{ \`{{ index .Cluster.Labels \"cluster.x-k8s.io\/provider\" }}\` }}\2/g" \
4548
"${ASSETS_DIR}/${FILE_NAME}"
4649

4750
kubectl create configmap "{{ .Values.hooks.clusterAutoscaler.crsStrategy.defaultInstallationConfigMap.name }}" --dry-run=client --output yaml \

‎make/addons.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
export CALICO_VERSION := v3.29.4
55
export CILIUM_VERSION := 1.17.4
66
export NODE_FEATURE_DISCOVERY_VERSION := 0.17.3
7-
export CLUSTER_AUTOSCALER_CHART_VERSION := 9.46.6
7+
export CLUSTER_AUTOSCALER_CHART_VERSION := 9.48.0
88
export AWS_EBS_CSI_CHART_VERSION := 2.44.0
99
export NUTANIX_STORAGE_CSI_CHART_VERSION := 3.3.4
1010
export LOCAL_PATH_CSI_CHART_VERSION := 0.0.31

0 commit comments

Comments
 (0)