diff --git a/test/framework/manifest/statefulset.go b/test/framework/manifest/statefulset.go new file mode 100644 index 00000000..42f98ec2 --- /dev/null +++ b/test/framework/manifest/statefulset.go @@ -0,0 +1,89 @@ +package manifest + +import ( + "github.com/aws/aws-sdk-go-v2/aws" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type StatefulSetBuilder struct { + namespace string + name string + serviceName string + replicas int32 + labels map[string]string + container corev1.Container + nodeSelector map[string]string + terminationGracePeriod int64 +} + +func NewDefaultStatefulSetBuilder() *StatefulSetBuilder { + return &StatefulSetBuilder{ + namespace: "default", + name: "statefulset", + serviceName: "statefulset", + replicas: 2, + labels: map[string]string{}, + container: NewBusyBoxContainerBuilder().Build(), + nodeSelector: map[string]string{"kubernetes.io/os": "linux"}, + terminationGracePeriod: 0, + } +} + +func (s *StatefulSetBuilder) Namespace(namespace string) *StatefulSetBuilder { + s.namespace = namespace + return s +} + +func (s *StatefulSetBuilder) Name(name string) *StatefulSetBuilder { + s.name = name + s.serviceName = name // typically same as name + return s +} + +func (s *StatefulSetBuilder) ServiceName(serviceName string) *StatefulSetBuilder { + s.serviceName = serviceName + return s +} + +func (s *StatefulSetBuilder) Replicas(replicas int32) *StatefulSetBuilder { + s.replicas = replicas + return s +} + +func (s *StatefulSetBuilder) PodLabel(key, value string) *StatefulSetBuilder { + s.labels[key] = value + return s +} + +func (s *StatefulSetBuilder) Container(container corev1.Container) *StatefulSetBuilder { + s.container = container + return s +} + +func (s *StatefulSetBuilder) Build() *appsv1.StatefulSet { + return &appsv1.StatefulSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: s.name, + Namespace: s.namespace, + }, + Spec: appsv1.StatefulSetSpec{ + ServiceName: s.serviceName, + Replicas: aws.Int32(s.replicas), + Selector: &metav1.LabelSelector{ + MatchLabels: s.labels, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: s.labels, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{s.container}, + NodeSelector: s.nodeSelector, + TerminationGracePeriodSeconds: aws.Int64(s.terminationGracePeriod), + }, + }, + }, + } +} diff --git a/test/integration/perpodsg/perpodsg_test.go b/test/integration/perpodsg/perpodsg_test.go old mode 100644 new mode 100755 index ef8f51e4..597f8eee --- a/test/integration/perpodsg/perpodsg_test.go +++ b/test/integration/perpodsg/perpodsg_test.go @@ -28,6 +28,7 @@ import ( sgpWrapper "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/resource/k8s/sgp" "github.com/aws/amazon-vpc-resource-controller-k8s/test/framework/utils" "github.com/samber/lo" + "sigs.k8s.io/controller-runtime/pkg/client" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -513,6 +514,101 @@ var _ = Describe("Branch ENI Pods", func() { }) }) }) + + Describe("Test Network Connectivity on Delete and Recreation of Pod", func() { + Context("creating statefulset with network connectivity", func() { + var resourceMap map[v1.ResourceName]resource.Quantity + var statefulSet *appsv1.StatefulSet + var container v1.Container + + BeforeEach(func() { + resourceMap = map[v1.ResourceName]resource.Quantity{ + config.ResourceNamePodENI: resource.MustParse("1"), + } + }) + + JustBeforeEach(func() { + container = manifest.NewBusyBoxContainerBuilder(). + Resources(v1.ResourceRequirements{ + Limits: resourceMap, + Requests: resourceMap, + }). + Command([]string{ "/bin/sh", "-c", + "while true; do if ping -c 1 google.com; then echo 'Successfully pinged google.com'; else echo 'Failed to ping google.com'; exit 1; fi; sleep 30; done", + }). + Name("network-test"). + Image("busybox"). + Build() + + statefulSet = manifest.NewDefaultStatefulSetBuilder(). + Namespace(namespace). + Name("network-test"). + PodLabel(podLabelKey, podLabelValue). + Container(container). + Build() + }) + + JustAfterEach(func() { + By("deleting the statefulset") + err = frameWork.K8sClient.Delete(ctx, statefulSet) + Expect(err).ToNot(HaveOccurred()) + }) + + Context("when statefulset is created with network connectivity requirements", func() { + It("should have all pods running with network access", func() { + By("creating security group policy") + sgpWrapper.CreateSecurityGroupPolicy(frameWork.K8sClient, ctx, securityGroupPolicy) + + By("creating statefulset") + err = frameWork.K8sClient.Create(ctx, statefulSet) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for statefulset pods to be ready") + Eventually(func() bool { + err := frameWork.K8sClient.Get(ctx, client.ObjectKey{ + Namespace: namespace, + Name: statefulSet.Name, + }, statefulSet) + if err != nil { + return false + } + return statefulSet.Status.ReadyReplicas == *statefulSet.Spec.Replicas + }, 300*time.Second, 5*time.Second).Should(BeTrue()) + + By("verifying network connectivity of all pods") + verify.VerifyNetworkingOfAllPodUsingENI(namespace, podLabelKey, podLabelValue, + securityGroups) + + By("force deleting one pod to verify recreation") + pods := &v1.PodList{} + err = frameWork.K8sClient.List(ctx, pods, client.InNamespace(namespace), + client.MatchingLabels(map[string]string{podLabelKey: podLabelValue})) + Expect(err).ToNot(HaveOccurred()) + Expect(pods.Items).ToNot(BeEmpty()) + + // Force delete the first pod + err = frameWork.K8sClient.Delete(ctx, &pods.Items[0], client.GracePeriodSeconds(0)) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for pod to be recreated") + Eventually(func() bool { + err := frameWork.K8sClient.Get(ctx, client.ObjectKey{ + Namespace: namespace, + Name: statefulSet.Name, + }, statefulSet) + if err != nil { + return false + } + return statefulSet.Status.ReadyReplicas == *statefulSet.Spec.Replicas + }, 300*time.Second, 5*time.Second).Should(BeTrue()) + + By("verifying network connectivity after pod recreation") + verify.VerifyNetworkingOfAllPodUsingENI(namespace, podLabelKey, podLabelValue, + securityGroups) + }) + }) + }) + }) }) func CreateServiceAccount(serviceAccount *v1.ServiceAccount) {