Skip to content

Commit 1009090

Browse files
jenshujenshu
authored
transformation e2e flakes (#10850)
Signed-off-by: Jenny Shu <jenny.shu@solo.io>
1 parent d42308c commit 1009090

File tree

4 files changed

+190
-72
lines changed

4 files changed

+190
-72
lines changed

‎.github/workflows/pr-kubernetes-tests.yaml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -26,22 +26,22 @@ jobs:
2626
# Above each test below, we document the latest date/time for the GitHub action step to run
2727
# NOTE: We use the GitHub action step time (as opposed to the `go test` time), because it is easier to capture
2828
test:
29-
# March 12, 2025: 7 minutes
29+
# March 18, 2025: 7 minutes
3030
- cluster-name: 'cluster-one'
3131
go-test-args: '-v -timeout=25m'
3232
go-test-run-regex: '^TestKgateway$$/^BasicRouting$$|^TestKgateway$$/^HTTPRouteServices$$|^TestKgateway$$/^TLSRouteServices$$'
3333
localstack: 'false'
34-
# Mar 12, 2025: 6 minutes
34+
# Mar 18, 2025: 7 minutes
3535
- cluster-name: 'cluster-two'
3636
go-test-args: '-v -timeout=25m'
3737
go-test-run-regex: '^TestKgateway$$/^Backends$$|^TestKgateway$$/^TCPRouteServices$$|^TestKgateway$$/^Transforms$$|^TestKgateway$$/^BackendTLSPolicies$$|^TestKgatewayWaypoint$$'
3838
localstack: 'false'
39-
# March 12, 2025: 7 minutes
39+
# March 18, 2025: 7 minutes
4040
- cluster-name: 'cluster-three'
4141
go-test-args: '-v -timeout=25m'
4242
go-test-run-regex: '^TestKgateway$$/^Deployer$$|^TestKgateway$$/^RouteDelegation$$|^TestKgateway$$/^Lambda$$'
4343
localstack: 'true'
44-
# March 13, 2025: 8 minutes
44+
# March 18, 2025: 6 minutes
4545
- cluster-name: 'cluster-ai'
4646
go-test-args: '-v -timeout=25m'
4747
go-test-run-regex: '^TestAIExtension'

‎test/kubernetes/e2e/features/transformation/suite.go

Lines changed: 90 additions & 67 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,8 @@ import (
44
"context"
55
"fmt"
66
"net/http"
7-
"time"
8-
97
"strings"
8+
"time"
109

1110
"github.com/onsi/gomega"
1211
"github.com/stretchr/testify/suite"
@@ -15,13 +14,14 @@ import (
1514
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1615
"sigs.k8s.io/controller-runtime/pkg/client"
1716

17+
envoyadmincli "github.com/kgateway-dev/kgateway/v2/pkg/utils/envoyutils/admincli"
1818
"github.com/kgateway-dev/kgateway/v2/pkg/utils/kubeutils"
1919
"github.com/kgateway-dev/kgateway/v2/pkg/utils/requestutils/curl"
2020
testmatchers "github.com/kgateway-dev/kgateway/v2/test/gomega/matchers"
21+
"github.com/kgateway-dev/kgateway/v2/test/helpers"
2122
"github.com/kgateway-dev/kgateway/v2/test/kubernetes/e2e"
23+
"github.com/kgateway-dev/kgateway/v2/test/kubernetes/e2e/defaults"
2224
testdefaults "github.com/kgateway-dev/kgateway/v2/test/kubernetes/e2e/defaults"
23-
24-
envoyadmincli "github.com/kgateway-dev/kgateway/v2/pkg/utils/envoyutils/admincli"
2525
)
2626

2727
var _ e2e.NewSuiteFunc = NewTestingSuite
@@ -35,6 +35,11 @@ type testingSuite struct {
3535
// testInstallation contains all the metadata/utilities necessary to execute a series of tests
3636
// against an installation of kgateway
3737
testInstallation *e2e.TestInstallation
38+
39+
// manifests shared by all tests
40+
commonManifests []string
41+
// resources from manifests shared by all tests
42+
commonResources []client.Object
3843
}
3944

4045
func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.TestingSuite {
@@ -44,41 +49,60 @@ func NewTestingSuite(ctx context.Context, testInst *e2e.TestInstallation) suite.
4449
}
4550
}
4651

47-
func (s *testingSuite) TestGatewayWithTransformedRoute() {
48-
manifests := []string{
52+
func (s *testingSuite) SetupSuite() {
53+
s.commonManifests = []string{
4954
testdefaults.CurlPodManifest,
5055
simpleServiceManifest,
5156
gatewayWithRouteManifest,
5257
}
53-
manifestObjects := []client.Object{
54-
testdefaults.CurlPod, // curl
55-
simpleSvc, // echo service
56-
proxyService, proxyServiceAccount, proxyDeployment, // proxy
58+
s.commonResources = []client.Object{
59+
// resources from curl manifest
60+
testdefaults.CurlPod,
61+
// resources from service manifest
62+
simpleSvc, simpleDeployment,
63+
// resources from gateway manifest
64+
gateway, route, routePolicy,
65+
// deployer-generated resources
66+
proxyDeployment, proxyService, proxyServiceAccount,
5767
}
5868

59-
s.T().Cleanup(func() {
60-
for _, manifest := range manifests {
61-
err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, manifest)
62-
s.Require().NoError(err)
63-
}
64-
s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, manifestObjects...)
65-
})
66-
67-
for _, manifest := range manifests {
69+
// set up common resources once
70+
for _, manifest := range s.commonManifests {
6871
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, manifest)
69-
s.Require().NoError(err)
72+
s.Require().NoError(err, "can apply "+manifest)
7073
}
71-
s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, manifestObjects...)
74+
s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, s.commonResources...)
7275

7376
// make sure pods are running
74-
s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, testdefaults.CurlPod.GetNamespace(), metav1.ListOptions{
75-
LabelSelector: "app.kubernetes.io/name=curl",
77+
s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, defaults.CurlPod.GetNamespace(), metav1.ListOptions{
78+
LabelSelector: defaults.CurlPodLabelSelector,
79+
})
80+
s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, simpleDeployment.GetNamespace(), metav1.ListOptions{
81+
LabelSelector: "app=backend-0,version=v1",
7682
})
77-
7883
s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, proxyObjectMeta.GetNamespace(), metav1.ListOptions{
79-
LabelSelector: "app.kubernetes.io/name=gw",
84+
LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", proxyObjectMeta.GetName()),
85+
})
86+
}
87+
88+
func (s *testingSuite) TearDownSuite() {
89+
// clean up common resources
90+
for _, manifest := range s.commonManifests {
91+
err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, manifest)
92+
s.Require().NoError(err, "can delete "+manifest)
93+
}
94+
s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, s.commonResources...)
95+
96+
// make sure pods are gone
97+
s.testInstallation.Assertions.EventuallyPodsNotExist(s.ctx, simpleDeployment.GetNamespace(), metav1.ListOptions{
98+
LabelSelector: "app=backend-0,version=v1",
99+
})
100+
s.testInstallation.Assertions.EventuallyPodsNotExist(s.ctx, proxyObjectMeta.GetNamespace(), metav1.ListOptions{
101+
LabelSelector: fmt.Sprintf("app.kubernetes.io/name=%s", proxyObjectMeta.GetName()),
80102
})
103+
}
81104

105+
func (s *testingSuite) TestGatewayWithTransformedRoute() {
82106
testCasess := []struct {
83107
name string
84108
opts []curl.Option
@@ -124,67 +148,66 @@ func (s *testingSuite) TestGatewayWithTransformedRoute() {
124148
}
125149

126150
func (s *testingSuite) TestGatewayRustformationsWithTransformedRoute() {
127-
manifests := []string{
128-
testdefaults.CurlPodManifest,
129-
simpleServiceManifest,
130-
gatewayWithRouteManifest,
131-
}
132-
manifestObjects := []client.Object{
133-
testdefaults.CurlPod, // curl
134-
simpleSvc, // echo service
135-
proxyService, proxyServiceAccount, proxyDeployment, // proxy
136-
}
137-
151+
// make a copy of the original controller deployment
138152
controllerDeploymentOriginal := &appsv1.Deployment{}
139153
err := s.testInstallation.ClusterContext.Client.Get(s.ctx, client.ObjectKey{
140154
Namespace: s.testInstallation.Metadata.InstallNamespace,
141-
Name: "kgateway",
155+
Name: helpers.DefaultKgatewayDeploymentName,
142156
}, controllerDeploymentOriginal)
143157
s.Assert().NoError(err, "has controller deploymnet")
144158

145-
controllerDeploy := controllerDeploymentOriginal.DeepCopy()
146-
// add the environment variable RUSTFORMATIONS to the controller deployment
147-
148-
env := append(controllerDeploy.Spec.Template.Spec.Containers[0].Env, corev1.EnvVar{
159+
// add the environment variable RUSTFORMATIONS to the modified controller deployment
160+
rustFormationsEnvVar := corev1.EnvVar{
149161
Name: "KGW_USE_RUST_FORMATIONS",
150162
Value: "true",
151-
})
152-
containers := controllerDeploy.Spec.Template.Spec.Containers
153-
containers[0].Env = env
154-
controllerDeploy.Spec.Template.Spec.Containers = containers
163+
}
164+
controllerDeployModified := controllerDeploymentOriginal.DeepCopy()
165+
controllerDeployModified.Spec.Template.Spec.Containers[0].Env = append(
166+
controllerDeployModified.Spec.Template.Spec.Containers[0].Env,
167+
rustFormationsEnvVar,
168+
)
155169

156-
// patch the actual deployment with the new environment variable
157-
err = s.testInstallation.ClusterContext.Client.Patch(s.ctx, controllerDeploy, client.MergeFrom(controllerDeploymentOriginal))
170+
// patch the deployment
171+
controllerDeployModified.ResourceVersion = ""
172+
err = s.testInstallation.ClusterContext.Client.Patch(s.ctx, controllerDeployModified, client.MergeFrom(controllerDeploymentOriginal))
158173
s.Assert().NoError(err, "patching controller deployment")
159174

160-
s.T().Cleanup(func() {
161-
for _, manifest := range manifests {
162-
err := s.testInstallation.Actions.Kubectl().DeleteFileSafe(s.ctx, manifest)
163-
s.Require().NoError(err)
164-
}
165-
s.testInstallation.Assertions.EventuallyObjectsNotExist(s.ctx, manifestObjects...)
166-
err = s.testInstallation.ClusterContext.Client.Patch(s.ctx, controllerDeploy, client.MergeFrom(controllerDeploy))
167-
s.Require().NoError(err)
168-
})
175+
// wait for the changes to be reflected in pod
176+
s.testInstallation.Assertions.EventuallyPodContainerContainsEnvVar(
177+
s.ctx,
178+
s.testInstallation.Metadata.InstallNamespace,
179+
metav1.ListOptions{
180+
LabelSelector: "app.kubernetes.io/name=kgateway",
181+
},
182+
helpers.KgatewayContainerName,
183+
rustFormationsEnvVar,
184+
)
169185

170-
for _, manifest := range manifests {
171-
err := s.testInstallation.Actions.Kubectl().ApplyFile(s.ctx, manifest)
186+
s.T().Cleanup(func() {
187+
// revert to original version of deployment
188+
controllerDeploymentOriginal.ResourceVersion = ""
189+
err = s.testInstallation.ClusterContext.Client.Patch(s.ctx, controllerDeploymentOriginal, client.MergeFrom(controllerDeployModified))
172190
s.Require().NoError(err)
173-
}
174-
s.testInstallation.Assertions.EventuallyObjectsExist(s.ctx, manifestObjects...)
175-
176-
// make sure pods are running
177-
s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, testdefaults.CurlPod.GetNamespace(), metav1.ListOptions{
178-
LabelSelector: "app.kubernetes.io/name=curl",
179-
})
180191

181-
s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, proxyObjectMeta.GetNamespace(), metav1.ListOptions{
182-
LabelSelector: "app.kubernetes.io/name=gw",
192+
// make sure the env var is removed
193+
s.testInstallation.Assertions.EventuallyPodContainerDoesNotContainEnvVar(
194+
s.ctx,
195+
s.testInstallation.Metadata.InstallNamespace,
196+
metav1.ListOptions{
197+
LabelSelector: "app.kubernetes.io/name=kgateway",
198+
},
199+
helpers.KgatewayContainerName,
200+
rustFormationsEnvVar.Name,
201+
)
183202
})
184203

204+
// wait for pods to be running again, since controller deployment was patched
185205
s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, s.testInstallation.Metadata.InstallNamespace, metav1.ListOptions{
186206
LabelSelector: "app.kubernetes.io/name=kgateway",
187207
})
208+
s.testInstallation.Assertions.EventuallyPodsRunning(s.ctx, proxyObjectMeta.GetNamespace(), metav1.ListOptions{
209+
LabelSelector: "app.kubernetes.io/name=gw",
210+
})
188211

189212
adminClient, closeFwd, err := envoyadmincli.NewPortForwardedClient(s.ctx, "deploy/"+proxyObjectMeta.Name, proxyObjectMeta.Namespace)
190213
s.Assert().NoError(err, "get admin cli for envoy")

‎test/kubernetes/e2e/features/transformation/types.go

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,9 @@ import (
66
appsv1 "k8s.io/api/apps/v1"
77
corev1 "k8s.io/api/core/v1"
88
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
9+
gwv1 "sigs.k8s.io/gateway-api/apis/v1"
910

11+
"github.com/kgateway-dev/kgateway/v2/api/v1alpha1"
1012
"github.com/kgateway-dev/kgateway/v2/pkg/utils/fsutils"
1113
)
1214

@@ -15,7 +17,26 @@ var (
1517
simpleServiceManifest = filepath.Join(fsutils.MustGetThisDir(), "testdata", "service.yaml")
1618
gatewayWithRouteManifest = filepath.Join(fsutils.MustGetThisDir(), "testdata", "gateway-with-transformed-route.yaml")
1719

18-
// objects
20+
// objects from gateway manifest
21+
gateway = &gwv1.Gateway{
22+
ObjectMeta: metav1.ObjectMeta{
23+
Name: "gw",
24+
Namespace: "default",
25+
},
26+
}
27+
route = &gwv1.HTTPRoute{
28+
ObjectMeta: metav1.ObjectMeta{
29+
Name: "example-route",
30+
Namespace: "default",
31+
},
32+
}
33+
routePolicy = &v1alpha1.RoutePolicy{
34+
ObjectMeta: metav1.ObjectMeta{
35+
Name: "requestresponse-transformer",
36+
Namespace: "default",
37+
},
38+
}
39+
// objects created by deployer after applying gateway manifest
1940
proxyObjectMeta = metav1.ObjectMeta{
2041
Name: "gw",
2142
Namespace: "default",
@@ -24,10 +45,17 @@ var (
2445
proxyService = &corev1.Service{ObjectMeta: proxyObjectMeta}
2546
proxyServiceAccount = &corev1.ServiceAccount{ObjectMeta: proxyObjectMeta}
2647

48+
// objects from service manifest
2749
simpleSvc = &corev1.Service{
2850
ObjectMeta: metav1.ObjectMeta{
2951
Name: "simple-svc",
3052
Namespace: "default",
3153
},
3254
}
55+
simpleDeployment = &appsv1.Deployment{
56+
ObjectMeta: metav1.ObjectMeta{
57+
Name: "backend-0",
58+
Namespace: "default",
59+
},
60+
}
3361
)

‎test/kubernetes/testutils/assertions/pods.go

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"time"
77

88
"github.com/onsi/gomega"
9+
"github.com/onsi/gomega/gstruct"
910
"github.com/onsi/gomega/types"
1011
corev1 "k8s.io/api/core/v1"
1112
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -89,3 +90,69 @@ func (p *Provider) EventuallyPodsNotExist(
8990
Should(gomega.Succeed(), fmt.Sprintf("pods matching %v in namespace %s should not be found in cluster",
9091
listOpt, podNamespace))
9192
}
93+
94+
// EventuallyPodContainerContainsEnvVar asserts that eventually all pods matching the given pod namespace and selector
95+
// have a container with the given container name and the given env var.
96+
func (p *Provider) EventuallyPodContainerContainsEnvVar(
97+
ctx context.Context,
98+
podNamespace string,
99+
podListOpt metav1.ListOptions,
100+
containerName string,
101+
envVar corev1.EnvVar,
102+
timeout ...time.Duration,
103+
) {
104+
currentTimeout, pollingInterval := helpers.GetTimeouts(timeout...)
105+
106+
p.Gomega.Eventually(func(g gomega.Gomega) {
107+
pods, err := p.clusterContext.Clientset.CoreV1().Pods(podNamespace).List(ctx, podListOpt)
108+
g.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to list pods")
109+
g.Expect(pods.Items).NotTo(gomega.BeEmpty(), "No pods found")
110+
for _, pod := range pods.Items {
111+
g.Expect(pod.Spec.Containers).To(gomega.ContainElement(
112+
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
113+
"Name": gomega.Equal(containerName),
114+
"Env": gomega.ContainElement(
115+
gomega.Equal(envVar),
116+
),
117+
}),
118+
))
119+
}
120+
}).
121+
WithTimeout(currentTimeout).
122+
WithPolling(pollingInterval).
123+
Should(gomega.Succeed(), fmt.Sprintf("Failed to match pod in namespace %s", podNamespace))
124+
}
125+
126+
// EventuallyPodContainerDoesNotContainEnvVar asserts that eventually no pods matching the given pod namespace and selector
127+
// have a container with the given name and env var with the given name.
128+
func (p *Provider) EventuallyPodContainerDoesNotContainEnvVar(
129+
ctx context.Context,
130+
podNamespace string,
131+
podListOpt metav1.ListOptions,
132+
containerName string,
133+
envVarName string,
134+
timeout ...time.Duration,
135+
) {
136+
currentTimeout, pollingInterval := helpers.GetTimeouts(timeout...)
137+
138+
p.Gomega.Eventually(func(g gomega.Gomega) {
139+
pods, err := p.clusterContext.Clientset.CoreV1().Pods(podNamespace).List(ctx, podListOpt)
140+
g.Expect(err).NotTo(gomega.HaveOccurred(), "Failed to list pods")
141+
g.Expect(pods.Items).NotTo(gomega.BeEmpty(), "No pods found")
142+
for _, pod := range pods.Items {
143+
g.Expect(pod.Spec.Containers).NotTo(gomega.ContainElement(
144+
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
145+
"Name": gomega.Equal(containerName),
146+
"Env": gomega.ContainElement(
147+
gstruct.MatchFields(gstruct.IgnoreExtras, gstruct.Fields{
148+
"Name": gomega.Equal(envVarName),
149+
}),
150+
),
151+
}),
152+
))
153+
}
154+
}).
155+
WithTimeout(currentTimeout).
156+
WithPolling(pollingInterval).
157+
Should(gomega.Succeed(), fmt.Sprintf("Failed to match pod in namespace %s", podNamespace))
158+
}

0 commit comments

Comments
 (0)