Add e2e tests for custom-error-pages

Signed-off-by: Ricardo Lopes <ricardoapl.dev@gmail.com>

Author: ricardoapl

docs/e2e-tests.md

@@ -283,6 +283,10 @@ Do not try to edit it manually.
 - [should return a self generated SSL certificate](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/ssl.go#L29)
 ### [[Default Backend] change default settings](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L30)
 - [should apply the annotation to the default backend](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/with_hosts.go#L38)
+### [[Default Backend] custom-error-pages](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L33)
+- [should export /metrics and /debug/vars by default](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L36)
+- [shouldn't export /metrics and /debug/vars when IS_METRICS_EXPORT is set to false](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L57)
+- [shouldn't export /metrics and /debug/vars when METRICS_PORT is set to a different port](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/defaultbackend/custom_error_pages.go#L80)
 ### [[Disable Leader] Routing works when leader election was disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L28)
 - [should create multiple ingress routings rules when leader election has disabled](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/disableleaderelection/disable_leader.go#L35)
 ### [[Endpointslices] long service name](https://github.com/kubernetes/ingress-nginx/tree/main//test/e2e/endpointslices/longname.go#L29)

test/e2e/CUSTOMERRORPAGES_IMAGE

@@ -0,0 +1 @@
+localhost/custom-error-pages:e2e

test/e2e/defaultbackend/custom_error_pages.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package defaultbackend
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/onsi/ginkgo/v2"
+	"github.com/stretchr/testify/assert"
+	appsv1 "k8s.io/api/apps/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"k8s.io/ingress-nginx/test/e2e/framework"
+)
+
+var _ = framework.IngressNginxDescribe("[Default Backend] custom-error-pages", func() {
+	f := framework.NewDefaultFramework("custom-error-pages")
+
+	ginkgo.It("should export /metrics and /debug/vars by default", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 200", "/metrics", http.StatusOK},
+			{"request to /debug/vars should return HTTP 200", "/debug/vars", http.StatusOK},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, nil)
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+
+	ginkgo.It("shouldn't export /metrics and /debug/vars when IS_METRICS_EXPORT is set to false", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 404", "/metrics", http.StatusNotFound},
+			{"request to /debug/vars should return HTTP 404", "/debug/vars", http.StatusNotFound},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, map[string]string{
+			"IS_METRICS_EXPORT": "false",
+		})
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+
+	ginkgo.It("shouldn't export /metrics and /debug/vars when METRICS_PORT is set to a different port", func() {
+		tt := []struct {
+			Name   string
+			Path   string
+			Status int
+		}{
+			{"request to /metrics should return HTTP 404", "/metrics", http.StatusNotFound},
+			{"request to /debug/vars should return HTTP 404", "/debug/vars", http.StatusNotFound},
+		}
+
+		setupIngressControllerWithCustomErrorPages(f, map[string]string{
+			"IS_METRICS_EXPORT": "true",
+			"METRICS_PORT":      "8081",
+		})
+
+		for _, t := range tt {
+			ginkgo.By(t.Name)
+			f.HTTPTestClient().
+				GET(t.Path).
+				Expect().
+				Status(t.Status)
+		}
+	})
+})
+
+func setupIngressControllerWithCustomErrorPages(f *framework.Framework, envVars map[string]string) {
+	f.NewCustomErrorPagesDeployment(framework.WithEnvVars(envVars))
+
+	err := f.UpdateIngressControllerDeployment(func(deployment *appsv1.Deployment) error {
+		args := deployment.Spec.Template.Spec.Containers[0].Args
+		args = append(args, fmt.Sprintf("--default-backend-service=%v/%v", f.Namespace, framework.CustomErrorPagesService))
+		deployment.Spec.Template.Spec.Containers[0].Args = args
+		_, err := f.KubeClientSet.AppsV1().Deployments(f.Namespace).Update(context.TODO(), deployment, metav1.UpdateOptions{})
+		return err
+	})
+	assert.Nil(ginkgo.GinkgoT(), err, "updating deployment")
+
+	f.WaitForNginxServer("_",
+		func(server string) bool {
+			return strings.Contains(server, `set $proxy_upstream_name "upstream-default-backend"`)
+		})
+}

test/e2e/framework/deployment.go

@@ -49,6 +49,12 @@ var HTTPBunImage = os.Getenv("HTTPBUN_IMAGE")
 // EchoImage is the default image to be used by the echo service
 const EchoImage = "registry.k8s.io/ingress-nginx/e2e-test-echo:v1.0.1@sha256:1cec65aa768720290d05d65ab1c297ca46b39930e56bc9488259f9114fcd30e2" //#nosec G101
 
+// CustomErrorPagesService name of the deployment for the custom-error-pages app
+const CustomErrorPagesService = "custom-error-pages"
+
+// CustomErrorPagesImage is the default image that is used to deploy custom-error-pages with the framework
+var CustomErrorPagesImage = os.Getenv("CUSTOMERRORPAGES_IMAGE")
+
 // TODO: change all Deployment functions to use these options
 // in order to reduce complexity and have a unified API across the
 // framework
@@ -58,6 +64,7 @@ type deploymentOptions struct {
 	image          string
 	replicas       int
 	svcAnnotations map[string]string
+	envVars        map[string]string
 }
 
 // WithDeploymentNamespace allows configuring the deployment's namespace
@@ -103,6 +110,74 @@ func WithImage(i string) func(*deploymentOptions) {
 	}
 }
 
+// WithEnvVars allows configuring environment variables for the deployment
+func WithEnvVars(e map[string]string) func(*deploymentOptions) {
+	return func(o *deploymentOptions) {
+		o.envVars = e
+	}
+}
+
+// NewCustomErrorPagesDeployment creates a new single replica deployment of the custom-error-pages server image in a particular namespace
+func (f *Framework) NewCustomErrorPagesDeployment(opts ...func(*deploymentOptions)) {
+	options := &deploymentOptions{
+		namespace: f.Namespace,
+		name:      CustomErrorPagesService,
+		replicas:  1,
+		image:     CustomErrorPagesImage,
+	}
+	for _, o := range opts {
+		o(options)
+	}
+
+	envVars := []corev1.EnvVar{}
+	for k, v := range options.envVars {
+		envVars = append(envVars, corev1.EnvVar{Name: k, Value: v})
+	}
+
+	f.EnsureDeployment(newDeployment(
+		options.name,
+		options.namespace,
+		options.image,
+		8080,
+		int32(options.replicas),
+		nil, nil,
+		envVars,
+		[]corev1.VolumeMount{},
+		[]corev1.Volume{},
+		false,
+	))
+
+	f.EnsureService(&corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        options.name,
+			Namespace:   options.namespace,
+			Annotations: options.svcAnnotations,
+		},
+		Spec: corev1.ServiceSpec{
+			Ports: []corev1.ServicePort{
+				{
+					Name:       "http",
+					Port:       8080,
+					TargetPort: intstr.FromInt(8080),
+					Protocol:   corev1.ProtocolTCP,
+				},
+			},
+			Selector: map[string]string{
+				"app": options.name,
+			},
+		},
+	})
+
+	err := WaitForEndpoints(
+		f.KubeClientSet,
+		DefaultTimeout,
+		options.name,
+		options.namespace,
+		options.replicas,
+	)
+	assert.Nil(ginkgo.GinkgoT(), err, "waiting for endpoints to become ready")
+}
+
 // NewEchoDeployment creates a new single replica deployment of the echo server image in a particular namespace
 func (f *Framework) NewEchoDeployment(opts ...func(*deploymentOptions)) {
 	options := &deploymentOptions{

test/e2e/run-e2e-suite.sh

@@ -52,6 +52,7 @@ fi
 BASEDIR=$(dirname "$0")
 NGINX_BASE_IMAGE=$(cat $BASEDIR/../../NGINX_BASE)
 HTTPBUN_IMAGE=$(cat $BASEDIR/HTTPBUN_IMAGE)
+CUSTOMERRORPAGES_IMAGE=$(cat $BASEDIR/CUSTOMERRORPAGES_IMAGE)
 
 echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}"
 kubectl create serviceaccount ingress-nginx-e2e || true
@@ -82,6 +83,7 @@ kubectl run --rm \
   --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \
   --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \
   --env="HTTPBUN_IMAGE=${HTTPBUN_IMAGE}" \
+  --env="CUSTOMERRORPAGES_IMAGE=${CUSTOMERRORPAGES_IMAGE}" \
   --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \
   e2e --image=nginx-ingress-controller:e2e
 

test/e2e/run-kind-e2e.sh

@@ -52,6 +52,7 @@ export KUBECONFIG="${KUBECONFIG:-$HOME/.kube/kind-config-$KIND_CLUSTER_NAME}"
 SKIP_INGRESS_IMAGE_CREATION="${SKIP_INGRESS_IMAGE_CREATION:-false}"
 SKIP_E2E_IMAGE_CREATION="${SKIP_E2E_IMAGE_CREATION:=false}"
 SKIP_CLUSTER_CREATION="${SKIP_CLUSTER_CREATION:-false}"
+SKIP_CUSTOMERRORPAGES_IMAGE_CREATION="${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION:-false}"
 
 if ! command -v kind --version &> /dev/null; then
   echo "kind is not installed. Use the package manager or visit the official site https://kind.sigs.k8s.io/"
@@ -104,12 +105,19 @@ if [ "${SKIP_E2E_IMAGE_CREATION}" = "false" ]; then
   echo "[dev-env] ..done building e2e-image"
 fi
 
+if [ "${SKIP_CUSTOMERRORPAGES_IMAGE_CREATION}" = "false" ]; then
+  echo "[dev-env] building custom-error-pages image"
+  REGISTRY=localhost NAME=custom-error-pages TAG=e2e make -C "${DIR}"/../../images build
+  echo "[dev-env] .. done building custom-error-pages image"
+fi
+
 # Preload images used in e2e tests
 KIND_WORKERS=$(kind get nodes --name="${KIND_CLUSTER_NAME}" | grep worker | awk '{printf (NR>1?",":"") $1}')
 
 echo "[dev-env] copying docker images to cluster..."
 
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" nginx-ingress-controller:e2e
 kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "${REGISTRY}"/controller:"${TAG}"
+kind load docker-image --name="${KIND_CLUSTER_NAME}" --nodes="${KIND_WORKERS}" "localhost/custom-error-pages:e2e"
 echo "[dev-env] running e2e tests..."
 make -C "${DIR}"/../../ e2e-test