Skip to content

Commit c0cf7c6

Browse files
authored
Merge pull request #11667 from sbueringer/pr-extend-scale-test
✨ Extend scale test and make ExtensionConfig name in RuntimeSDK test configurable
2 parents a090145 + 94af8e9 commit c0cf7c6

File tree

8 files changed

+331
-181
lines changed

8 files changed

+331
-181
lines changed

test/e2e/cluster_upgrade_runtimesdk.go

Lines changed: 38 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"fmt"
2222
"os"
2323
"path/filepath"
24+
"strconv"
2425
"strings"
2526
"time"
2627

@@ -90,9 +91,15 @@ type ClusterUpgradeWithRuntimeSDKSpecInput struct {
9091
// If not specified, this is a no-op.
9192
PostUpgrade func(managementClusterProxy framework.ClusterProxy, workloadClusterNamespace, workloadClusterName string)
9293

94+
// ExtensionConfigName is the name of the ExtensionConfig. Defaults to "k8s-upgrade-with-runtimesdk".
95+
// This value is provided to clusterctl as "EXTENSION_CONFIG_NAME" variable and can be used to template the
96+
// name of the ExtensionConfig into the ClusterClass.
97+
ExtensionConfigName string
98+
9399
// ExtensionServiceNamespace is the namespace where the service for the Runtime SDK is located
94100
// and is used to configure in the test-namespace scoped ExtensionConfig.
95101
ExtensionServiceNamespace string
102+
96103
// ExtensionServiceName is the name of the service to configure in the test-namespace scoped ExtensionConfig.
97104
ExtensionServiceName string
98105
}
@@ -133,6 +140,9 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
133140

134141
Expect(input.ExtensionServiceNamespace).ToNot(BeEmpty())
135142
Expect(input.ExtensionServiceName).ToNot(BeEmpty())
143+
if input.ExtensionConfigName == "" {
144+
input.ExtensionConfigName = specName
145+
}
136146

137147
if input.ControlPlaneMachineCount == nil {
138148
controlPlaneMachineCount = 1
@@ -161,8 +171,11 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
161171

162172
By("Deploy Test Extension ExtensionConfig")
163173

174+
// In this test we are defaulting all handlers to blocking because we expect the handlers to block the
175+
// cluster lifecycle by default. Setting defaultAllHandlersToBlocking to true enforces that the test-extension
176+
// automatically creates the ConfigMap with blocking preloaded responses.
164177
Expect(input.BootstrapClusterProxy.GetClient().Create(ctx,
165-
extensionConfig(specName, namespace.Name, input.ExtensionServiceNamespace, input.ExtensionServiceName))).
178+
extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true, namespace.Name))).
166179
To(Succeed(), "Failed to create the extension config")
167180

168181
By("Creating a workload cluster; creation waits for BeforeClusterCreateHook to gate the operation")
@@ -177,6 +190,11 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
177190
infrastructureProvider = *input.InfrastructureProvider
178191
}
179192

193+
variables := map[string]string{
194+
// This is used to template the name of the ExtensionConfig into the ClusterClass.
195+
"EXTENSION_CONFIG_NAME": input.ExtensionConfigName,
196+
}
197+
180198
clusterctl.ApplyClusterTemplateAndWait(ctx, clusterctl.ApplyClusterTemplateAndWaitInput{
181199
ClusterProxy: input.BootstrapClusterProxy,
182200
ConfigCluster: clusterctl.ConfigClusterInput{
@@ -190,6 +208,7 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
190208
KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeFrom),
191209
ControlPlaneMachineCount: ptr.To[int64](controlPlaneMachineCount),
192210
WorkerMachineCount: ptr.To[int64](workerMachineCount),
211+
ClusterctlVariables: variables,
193212
},
194213
PreWaitForCluster: func() {
195214
beforeClusterCreateTestHandler(ctx,
@@ -304,8 +323,8 @@ func ClusterUpgradeWithRuntimeSDKSpec(ctx context.Context, inputGetter func() Cl
304323
if !input.SkipCleanup {
305324
// Delete the extensionConfig first to ensure the BeforeDeleteCluster hook doesn't block deletion.
306325
Eventually(func() error {
307-
return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(specName, namespace.Name, input.ExtensionServiceNamespace, input.ExtensionServiceName))
308-
}, 10*time.Second, 1*time.Second).Should(Succeed(), "delete extensionConfig failed")
326+
return input.BootstrapClusterProxy.GetClient().Delete(ctx, extensionConfig(input.ExtensionConfigName, input.ExtensionServiceNamespace, input.ExtensionServiceName, true, namespace.Name))
327+
}, 10*time.Second, 1*time.Second).Should(Succeed(), "Deleting ExtensionConfig failed")
309328

310329
Byf("Deleting cluster %s", klog.KObj(clusterResources.Cluster))
311330
// While https://github.com/kubernetes-sigs/cluster-api/issues/2955 is addressed in future iterations, there is a chance
@@ -429,8 +448,8 @@ func machineSetPreflightChecksTestHandler(ctx context.Context, c client.Client,
429448
// We make sure this cluster-wide object does not conflict with others by using a random generated
430449
// name and a NamespaceSelector selecting on the namespace of the current test.
431450
// Thus, this object is "namespaced" to the current test even though it's a cluster-wide object.
432-
func extensionConfig(name, namespace, extensionServiceNamespace, extensionServiceName string) *runtimev1.ExtensionConfig {
433-
return &runtimev1.ExtensionConfig{
451+
func extensionConfig(name, extensionServiceNamespace, extensionServiceName string, defaultAllHandlersToBlocking bool, namespaces ...string) *runtimev1.ExtensionConfig {
452+
cfg := &runtimev1.ExtensionConfig{
434453
ObjectMeta: metav1.ObjectMeta{
435454
// Note: We have to use a constant name here as we have to be able to reference it in the ClusterClass
436455
// when configuring external patches.
@@ -448,25 +467,24 @@ func extensionConfig(name, namespace, extensionServiceNamespace, extensionServic
448467
Namespace: extensionServiceNamespace,
449468
},
450469
},
451-
NamespaceSelector: &metav1.LabelSelector{
452-
// Note: we are limiting the test extension to be used by the namespace where the test is run.
453-
MatchExpressions: []metav1.LabelSelectorRequirement{
454-
{
455-
Key: "kubernetes.io/metadata.name",
456-
Operator: metav1.LabelSelectorOpIn,
457-
Values: []string{namespace},
458-
},
459-
},
460-
},
461470
Settings: map[string]string{
462-
// In the E2E test we are defaulting all handlers to blocking because cluster_upgrade_runtimesdk_test
463-
// expects the handlers to block the cluster lifecycle by default.
464-
// Setting this value to true enforces that the test-extension automatically creates the ConfigMap with
465-
// blocking preloaded responses.
466-
"defaultAllHandlersToBlocking": "true",
471+
"defaultAllHandlersToBlocking": strconv.FormatBool(defaultAllHandlersToBlocking),
467472
},
468473
},
469474
}
475+
if len(namespaces) > 0 {
476+
cfg.Spec.NamespaceSelector = &metav1.LabelSelector{
477+
// Note: we are limiting the test extension to be used by the namespace where the test is run.
478+
MatchExpressions: []metav1.LabelSelectorRequirement{
479+
{
480+
Key: "kubernetes.io/metadata.name",
481+
Operator: metav1.LabelSelectorOpIn,
482+
Values: namespaces,
483+
},
484+
},
485+
}
486+
}
487+
return cfg
470488
}
471489

472490
// Check that each hook in hooks has been called at least once by checking if its actualResponseStatus is in the hook response configmap.

test/e2e/data/infrastructure-docker/main/clusterclass-quick-start-runtimesdk.yaml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -54,9 +54,9 @@ spec:
5454
patches:
5555
- name: test-patch
5656
external:
57-
generateExtension: generate-patches.k8s-upgrade-with-runtimesdk
58-
validateExtension: validate-topology.k8s-upgrade-with-runtimesdk
59-
discoverVariablesExtension: discover-variables.k8s-upgrade-with-runtimesdk
57+
generateExtension: generate-patches.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
58+
validateExtension: validate-topology.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
59+
discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-"k8s-upgrade-with-runtimesdk"}
6060
---
6161
apiVersion: infrastructure.cluster.x-k8s.io/v1beta1
6262
kind: DockerClusterTemplate

test/e2e/data/infrastructure-inmemory/main/bases/cluster-with-topology.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ spec:
1212
serviceDomain: ${SERVICE_DOMAIN:="cluster.local"}
1313
topology:
1414
class: in-memory
15+
classNamespace: ${NAMESPACE}
1516
version: ${KUBERNETES_VERSION}
1617
controlPlane:
1718
replicas: ${CONTROL_PLANE_MACHINE_COUNT}
@@ -20,3 +21,8 @@ spec:
2021
- class: default-worker
2122
name: md-0
2223
replicas: ${WORKER_MACHINE_COUNT}
24+
variables:
25+
- name: kubeadmControlPlaneMaxSurge
26+
value: "1"
27+
- name: imageRepository
28+
value: "kindest"
Lines changed: 67 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -1,56 +1,3 @@
1-
apiVersion: cluster.x-k8s.io/v1beta1
2-
kind: ClusterClass
3-
metadata:
4-
name: in-memory
5-
spec:
6-
controlPlane:
7-
metadata:
8-
annotations:
9-
machineInfrastructure:
10-
ref:
11-
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
12-
kind: InMemoryMachineTemplate
13-
name: in-memory-control-plane
14-
ref:
15-
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
16-
kind: KubeadmControlPlaneTemplate
17-
name: in-memory-control-plane
18-
machineHealthCheck:
19-
unhealthyConditions:
20-
- type: Ready
21-
status: Unknown
22-
timeout: 300s
23-
- type: Ready
24-
status: "False"
25-
timeout: 300s
26-
infrastructure:
27-
ref:
28-
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
29-
kind: InMemoryClusterTemplate
30-
name: in-memory-cluster
31-
workers:
32-
machineDeployments:
33-
- class: default-worker
34-
template:
35-
bootstrap:
36-
ref:
37-
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
38-
kind: KubeadmConfigTemplate
39-
name: in-memory-default-worker-bootstraptemplate
40-
infrastructure:
41-
ref:
42-
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
43-
kind: InMemoryMachineTemplate
44-
name: in-memory-default-worker-machinetemplate
45-
machineHealthCheck:
46-
unhealthyConditions:
47-
- type: Ready
48-
status: Unknown
49-
timeout: 300s
50-
- type: Ready
51-
status: "False"
52-
timeout: 300s
53-
---
541
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
552
kind: InMemoryClusterTemplate
563
metadata:
@@ -95,19 +42,19 @@ spec:
9542
behaviour:
9643
vm:
9744
provisioning:
98-
startupDuration: "30s"
45+
startupDuration: "10s"
9946
startupJitter: "0.2"
10047
node:
10148
provisioning:
102-
startupDuration: "10s"
49+
startupDuration: "2s"
10350
startupJitter: "0.2"
10451
apiServer:
10552
provisioning:
106-
startupDuration: "10s"
53+
startupDuration: "2s"
10754
startupJitter: "0.2"
10855
etcd:
10956
provisioning:
110-
startupDuration: "10s"
57+
startupDuration: "2s"
11158
startupJitter: "0.2"
11259
---
11360
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
@@ -120,19 +67,19 @@ spec:
12067
behaviour:
12168
vm:
12269
provisioning:
123-
startupDuration: "30s"
70+
startupDuration: "10s"
12471
startupJitter: "0.2"
12572
node:
12673
provisioning:
127-
startupDuration: "10s"
74+
startupDuration: "2s"
12875
startupJitter: "0.2"
12976
apiServer:
13077
provisioning:
131-
startupDuration: "10s"
78+
startupDuration: "2s"
13279
startupJitter: "0.2"
13380
etcd:
13481
provisioning:
135-
startupDuration: "10s"
82+
startupDuration: "2s"
13683
startupJitter: "0.2"
13784
---
13885
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
@@ -146,4 +93,62 @@ spec:
14693
nodeRegistration:
14794
criSocket: unix:///var/run/containerd/containerd.sock
14895
kubeletExtraArgs:
149-
eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
96+
eviction-hard: nodefs.available<0%,nodefs.inodesFree<0%,imagefs.available<0%
97+
---
98+
apiVersion: cluster.x-k8s.io/v1beta1
99+
kind: ClusterClass
100+
metadata:
101+
name: in-memory
102+
spec:
103+
controlPlane:
104+
metadata:
105+
annotations:
106+
machineInfrastructure:
107+
ref:
108+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
109+
kind: InMemoryMachineTemplate
110+
name: in-memory-control-plane
111+
ref:
112+
apiVersion: controlplane.cluster.x-k8s.io/v1beta1
113+
kind: KubeadmControlPlaneTemplate
114+
name: in-memory-control-plane
115+
machineHealthCheck:
116+
unhealthyConditions:
117+
- type: Ready
118+
status: Unknown
119+
timeout: 300s
120+
- type: Ready
121+
status: "False"
122+
timeout: 300s
123+
infrastructure:
124+
ref:
125+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
126+
kind: InMemoryClusterTemplate
127+
name: in-memory-cluster
128+
workers:
129+
machineDeployments:
130+
- class: default-worker
131+
template:
132+
bootstrap:
133+
ref:
134+
apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
135+
kind: KubeadmConfigTemplate
136+
name: in-memory-default-worker-bootstraptemplate
137+
infrastructure:
138+
ref:
139+
apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
140+
kind: InMemoryMachineTemplate
141+
name: in-memory-default-worker-machinetemplate
142+
machineHealthCheck:
143+
unhealthyConditions:
144+
- type: Ready
145+
status: Unknown
146+
timeout: 300s
147+
- type: Ready
148+
status: "False"
149+
timeout: 300s
150+
patches:
151+
- name: test-patch
152+
external:
153+
generateExtension: generate-patches.${EXTENSION_CONFIG_NAME:-"scale"}
154+
discoverVariablesExtension: discover-variables.${EXTENSION_CONFIG_NAME:-"scale"}

0 commit comments

Comments
 (0)