diff --git a/api/v1alpha1/perconaservermysql_types.go b/api/v1alpha1/perconaservermysql_types.go index 113f50d3e..11d30084f 100644 --- a/api/v1alpha1/perconaservermysql_types.go +++ b/api/v1alpha1/perconaservermysql_types.go @@ -414,6 +414,8 @@ type MySQLRouterSpec struct { Expose ServiceExpose `json:"expose,omitempty"` + Ports []corev1.ServicePort `json:"ports,omitempty"` + PodSpec `json:",inline"` } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index d4e73d039..bb3ef1cb7 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -338,6 +338,13 @@ func (in *HAProxySpec) DeepCopy() *HAProxySpec { func (in *MySQLRouterSpec) DeepCopyInto(out *MySQLRouterSpec) { *out = *in in.Expose.DeepCopyInto(&out.Expose) + if in.Ports != nil { + in, out := &in.Ports, &out.Ports + *out = make([]corev1.ServicePort, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } in.PodSpec.DeepCopyInto(&out.PodSpec) } diff --git a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml index 16fc162ea..aacceb9b2 100644 --- a/config/crd/bases/ps.percona.com_perconaservermysqls.yaml +++ b/config/crd/bases/ps.percona.com_perconaservermysqls.yaml @@ -8496,6 +8496,31 @@ spec: type: string type: object type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array priorityClassName: type: string readinessProbe: diff --git a/deploy/bundle.yaml b/deploy/bundle.yaml index 549454ee4..14337bbe0 100644 --- a/deploy/bundle.yaml +++ b/deploy/bundle.yaml @@ -10419,6 +10419,31 @@ spec: type: string type: object type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array priorityClassName: type: string readinessProbe: diff --git a/deploy/cr.yaml b/deploy/cr.yaml index 85a7774b6..340333088 100644 --- a/deploy/cr.yaml +++ b/deploy/cr.yaml @@ -293,7 +293,34 @@ spec: # - "secret-1" # - "secret-2" # initImage: perconalab/percona-server-mysql-operator:main - +# ports: +# - name: http +# port: 8443 +# targetPort: 0 +# - name: rw-default +# port: 3306 +# targetPort: 6446 +# - name: read-write +# port: 6446 +# targetPort: 0 +# - name: read-only +# port: 6447 +# targetPort: 0 +# - name: x-read-write +# port: 6448 +# targetPort: 0 +# - name: x-read-only +# port: 6449 +# targetPort: 0 +# - name: x-default +# port: 33060 +# targetPort: 0 +# - name: rw-admin +# port: 33062 +# targetPort: 0 +# - name: custom-port +# port: 1111 +# targetPort: 6446 size: 3 resources: diff --git a/deploy/crd.yaml b/deploy/crd.yaml index c95ac6cc0..3a0dc4926 100644 --- a/deploy/crd.yaml +++ b/deploy/crd.yaml @@ -10419,6 +10419,31 @@ spec: type: string type: object type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array priorityClassName: type: string readinessProbe: diff --git a/deploy/cw-bundle.yaml b/deploy/cw-bundle.yaml index c7a2daa0a..d3cbff033 100644 --- a/deploy/cw-bundle.yaml +++ b/deploy/cw-bundle.yaml @@ -10419,6 +10419,31 @@ spec: type: string type: object type: object + ports: + items: + properties: + appProtocol: + type: string + name: + type: string + nodePort: + format: int32 + type: integer + port: + format: int32 + type: integer + protocol: + default: TCP + type: string + targetPort: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: array priorityClassName: type: string readinessProbe: diff --git a/e2e-tests/tests/gr-one-pod/01-assert.yaml b/e2e-tests/tests/gr-one-pod/01-assert.yaml index 0e772cd31..297800d50 100644 --- a/e2e-tests/tests/gr-one-pod/01-assert.yaml +++ b/e2e-tests/tests/gr-one-pod/01-assert.yaml @@ -28,6 +28,47 @@ metadata: name: gr-one-pod-router spec: replicas: 1 + template: + spec: + containers: + - args: + - mysqlrouter + - -c + - /tmp/router/mysqlrouter.conf + command: + - /opt/percona/router-entrypoint.sh + env: + - name: MYSQL_SERVICE_NAME + value: gr-one-pod-mysql + name: router + ports: + - containerPort: 8443 + name: http + protocol: TCP + - containerPort: 6446 + name: rw-default + protocol: TCP + - containerPort: 6446 + name: read-write + protocol: TCP + - containerPort: 6447 + name: read-only + protocol: TCP + - containerPort: 6448 + name: x-read-write + protocol: TCP + - containerPort: 6449 + name: x-read-only + protocol: TCP + - containerPort: 33060 + name: x-default + protocol: TCP + - containerPort: 33062 + name: rw-admin + protocol: TCP + - containerPort: 3333 + name: custom-port + protocol: TCP status: availableReplicas: 1 observedGeneration: 1 @@ -59,3 +100,67 @@ status: size: 1 state: ready state: ready +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: router + app.kubernetes.io/instance: gr-one-pod + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server + name: gr-one-pod-router +spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: http + port: 8443 + protocol: TCP + targetPort: 8443 + - name: rw-default + port: 1111 + protocol: TCP + targetPort: 6446 + - name: read-write + port: 6446 + protocol: TCP + targetPort: 6446 + - name: read-only + port: 6447 + protocol: TCP + targetPort: 6447 + - name: x-read-write + port: 6448 + protocol: TCP + targetPort: 6448 + - name: x-read-only + port: 6449 + protocol: TCP + targetPort: 6449 + - name: x-default + port: 33060 + protocol: TCP + targetPort: 33060 + - name: rw-admin + port: 33062 + protocol: TCP + targetPort: 33062 + - name: custom-port + port: 2222 + protocol: TCP + targetPort: 3333 + selector: + app.kubernetes.io/component: router + app.kubernetes.io/instance: gr-one-pod + app.kubernetes.io/managed-by: percona-server-operator + app.kubernetes.io/name: percona-server + app.kubernetes.io/part-of: percona-server + sessionAffinity: None + type: ClusterIP +status: + loadBalancer: {} + diff --git a/e2e-tests/tests/gr-one-pod/01-create-cluster.yaml b/e2e-tests/tests/gr-one-pod/01-create-cluster.yaml index 5cfe856a0..f6240fcde 100644 --- a/e2e-tests/tests/gr-one-pod/01-create-cluster.yaml +++ b/e2e-tests/tests/gr-one-pod/01-create-cluster.yaml @@ -17,6 +17,11 @@ commands: | yq eval ".spec.mysql.size=1" - \ | yq eval ".spec.proxy.haproxy.enabled=false" - \ | yq eval ".spec.proxy.router.size=1" - \ + | yq eval '.spec.proxy.router.ports[0].name="rw-default"' - \ + | yq eval ".spec.proxy.router.ports[0].port=1111" - \ + | yq eval '.spec.proxy.router.ports[1].name="custom-port"' - \ + | yq eval ".spec.proxy.router.ports[1].port=2222" - \ + | yq eval ".spec.proxy.router.ports[1].targetPort=3333" - \ | yq eval ".spec.orchestrator.enabled=false" - \ | yq eval '.spec.backup.storages.minio.type="s3"' - \ | yq eval '.spec.backup.storages.minio.s3.bucket="operator-testing"' - \ diff --git a/e2e-tests/tests/gr-one-pod/02-write-data.yaml b/e2e-tests/tests/gr-one-pod/02-write-data.yaml index 72f7d6e07..5df03b902 100644 --- a/e2e-tests/tests/gr-one-pod/02-write-data.yaml +++ b/e2e-tests/tests/gr-one-pod/02-write-data.yaml @@ -9,8 +9,8 @@ commands: run_mysql \ "CREATE DATABASE IF NOT EXISTS myDB; CREATE TABLE IF NOT EXISTS myDB.myTable (id int PRIMARY KEY)" \ - "-h $(get_router_service $(get_cluster_name)) -uroot -proot_password" + "-h $(get_router_service $(get_cluster_name)) -P1111 -uroot -proot_password" run_mysql \ "INSERT myDB.myTable (id) VALUES (100500)" \ - "-h $(get_router_service $(get_cluster_name)) -uroot -proot_password" + "-h $(get_router_service $(get_cluster_name)) -P1111 -uroot -proot_password" diff --git a/e2e-tests/tests/gr-one-pod/04-delete-data.yaml b/e2e-tests/tests/gr-one-pod/04-delete-data.yaml index b2d37a302..ba4fe17bf 100644 --- a/e2e-tests/tests/gr-one-pod/04-delete-data.yaml +++ b/e2e-tests/tests/gr-one-pod/04-delete-data.yaml @@ -9,7 +9,7 @@ commands: run_mysql \ "TRUNCATE TABLE myDB.myTable" \ - "-h $(get_router_service $(get_cluster_name)) -uroot -proot_password" + "-h $(get_router_service $(get_cluster_name)) -P1111 -uroot -proot_password" - data=$(run_mysql "SELECT * FROM myDB.myTable" "-h $(get_router_service $(get_cluster_name)) -uroot -proot_password") + data=$(run_mysql "SELECT * FROM myDB.myTable" "-h $(get_router_service $(get_cluster_name)) -P1111 -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 04-delete-data-minio --from-literal=data="${data}" diff --git a/e2e-tests/tests/gr-one-pod/06-read-data.yaml b/e2e-tests/tests/gr-one-pod/06-read-data.yaml index cbc2dfa93..481985c9d 100644 --- a/e2e-tests/tests/gr-one-pod/06-read-data.yaml +++ b/e2e-tests/tests/gr-one-pod/06-read-data.yaml @@ -10,6 +10,6 @@ commands: wait_cluster_consistency_gr "${test_name}" "1" "1" - data=$(run_mysql "SELECT * FROM myDB.myTable" "-h $(get_router_service $(get_cluster_name)) -uroot -proot_password") + data=$(run_mysql "SELECT * FROM myDB.myTable" "-h $(get_router_service $(get_cluster_name)) -P1111 -uroot -proot_password") kubectl create configmap -n "${NAMESPACE}" 06-read-data-minio --from-literal=data="${data}" timeout: 120 diff --git a/pkg/router/router.go b/pkg/router/router.go index a34f30522..46c038872 100644 --- a/pkg/router/router.go +++ b/pkg/router/router.go @@ -2,6 +2,7 @@ package router import ( "fmt" + "slices" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" @@ -27,14 +28,14 @@ const ( ) const ( - PortHTTP = 8443 - PortRWDefault = 3306 - PortReadWrite = 6446 - PortReadOnly = 6447 - PortXReadWrite = 6448 - PortXReadOnly = 6449 - PortXDefault = 33060 - PortRWAdmin = 33062 + portHTTP = 8443 + portRWDefault = 3306 + portReadWrite = 6446 + portReadOnly = 6447 + portXReadWrite = 6448 + portXReadOnly = 6449 + portXDefault = 33060 + portRWAdmin = 33062 ) func Name(cr *apiv1alpha1.PerconaServerMySQL) string { @@ -69,7 +70,7 @@ func Service(cr *apiv1alpha1.PerconaServerMySQL) *corev1.Service { externalTrafficPolicy = expose.ExternalTrafficPolicy } - return &corev1.Service{ + s := &corev1.Service{ TypeMeta: metav1.TypeMeta{ APIVersion: "v1", Kind: "Service", @@ -81,52 +82,16 @@ func Service(cr *apiv1alpha1.PerconaServerMySQL) *corev1.Service { Annotations: expose.Annotations, }, Spec: corev1.ServiceSpec{ - Type: expose.Type, - Ports: []corev1.ServicePort{ - // do not change the port order - // 8443 port should be the first in service, see K8SPS-132 task - { - Name: "http", - Port: int32(PortHTTP), - }, - { - Name: "rw-default", - Port: int32(PortRWDefault), - TargetPort: intstr.IntOrString{ - IntVal: PortReadWrite, - }, - }, - { - Name: "read-write", - Port: int32(PortReadWrite), - }, - { - Name: "read-only", - Port: int32(PortReadOnly), - }, - { - Name: "x-read-write", - Port: int32(PortXReadWrite), - }, - { - Name: "x-read-only", - Port: int32(PortXReadOnly), - }, - { - Name: "x-default", - Port: int32(PortXDefault), - }, - { - Name: "rw-admin", - Port: int32(PortRWAdmin), - }, - }, + Type: expose.Type, + Ports: ports(cr.Spec.Proxy.Router.Ports), Selector: labels, LoadBalancerSourceRanges: loadBalancerSourceRanges, InternalTrafficPolicy: expose.InternalTrafficPolicy, ExternalTrafficPolicy: externalTrafficPolicy, }, } + + return s } func Deployment(cr *apiv1alpha1.PerconaServerMySQL, initImage, configHash, tlsHash string) *appsv1.Deployment { @@ -256,6 +221,79 @@ func containers(cr *apiv1alpha1.PerconaServerMySQL) []corev1.Container { return []corev1.Container{routerContainer(cr)} } +func ports(sPorts []corev1.ServicePort) []corev1.ServicePort { + defaultPorts := []corev1.ServicePort{ + // do not change the port order + // 8443 port should be the first in service, see K8SPS-132 task + { + Name: "http", + Port: int32(portHTTP), + }, + { + Name: "rw-default", + Port: int32(portRWDefault), + TargetPort: intstr.IntOrString{ + IntVal: portReadWrite, + }, + }, + { + Name: "read-write", + Port: int32(portReadWrite), + }, + { + Name: "read-only", + Port: int32(portReadOnly), + }, + { + Name: "x-read-write", + Port: int32(portXReadWrite), + }, + { + Name: "x-read-only", + Port: int32(portXReadOnly), + }, + { + Name: "x-default", + Port: int32(portXDefault), + }, + { + Name: "rw-admin", + Port: int32(portRWAdmin), + }, + } + if len(sPorts) == 0 { + return defaultPorts + } + + specifiedPorts := make([]corev1.ServicePort, len(sPorts)) + copy(specifiedPorts, sPorts) + + ports := []corev1.ServicePort{} + for _, defaultPort := range defaultPorts { + idx := slices.IndexFunc(specifiedPorts, func(port corev1.ServicePort) bool { + return port.Name == defaultPort.Name + }) + if idx == -1 { + ports = append(ports, defaultPort) + continue + } + + modifiedPort := specifiedPorts[idx] + if modifiedPort.Port == 0 { + modifiedPort.Port = defaultPort.Port + } + if modifiedPort.TargetPort.IntValue() == 0 { + modifiedPort.TargetPort = defaultPort.TargetPort + } + ports = append(ports, modifiedPort) + specifiedPorts = slices.Delete(specifiedPorts, idx, idx+1) + } + + ports = append(ports, specifiedPorts...) + + return ports +} + func routerContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container { spec := cr.Spec.Proxy.Router @@ -267,43 +305,13 @@ func routerContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container { } env = append(env, spec.Env...) - return corev1.Container{ + c := corev1.Container{ Name: AppName, Image: spec.Image, ImagePullPolicy: spec.ImagePullPolicy, Resources: spec.Resources, Env: env, EnvFrom: spec.EnvFrom, - Ports: []corev1.ContainerPort{ - { - Name: "http", - ContainerPort: int32(PortHTTP), - }, - { - Name: "read-write", - ContainerPort: int32(PortReadWrite), - }, - { - Name: "read-only", - ContainerPort: int32(PortReadOnly), - }, - { - Name: "x-read-write", - ContainerPort: int32(PortXReadWrite), - }, - { - Name: "x-read-only", - ContainerPort: int32(PortXReadOnly), - }, - { - Name: "x-default", - ContainerPort: int32(PortXDefault), - }, - { - Name: "rw-admin", - ContainerPort: int32(PortRWAdmin), - }, - }, VolumeMounts: []corev1.VolumeMount{ { Name: apiv1alpha1.BinVolumeName, @@ -330,4 +338,19 @@ func routerContainer(cr *apiv1alpha1.PerconaServerMySQL) corev1.Container { StartupProbe: k8s.ExecProbe(spec.StartupProbe, []string{"/opt/percona/router_startup_check.sh"}), ReadinessProbe: k8s.ExecProbe(spec.ReadinessProbe, []string{"/opt/percona/router_readiness_check.sh"}), } + + for _, servicePort := range ports(cr.Spec.Proxy.Router.Ports) { + containerPort := servicePort.Port + if targetPort := servicePort.TargetPort.IntValue(); targetPort != 0 { + containerPort = int32(targetPort) + } + + c.Ports = append(c.Ports, corev1.ContainerPort{ + Name: servicePort.Name, + ContainerPort: containerPort, + Protocol: servicePort.Protocol, + }) + } + + return c } diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go index 9d46d83fd..d0a1a9405 100644 --- a/pkg/router/router_test.go +++ b/pkg/router/router_test.go @@ -1,16 +1,204 @@ package router import ( + "reflect" "testing" - "github.com/stretchr/testify/assert" + "github.com/google/go-cmp/cmp" corev1 "k8s.io/api/core/v1" - "k8s.io/utils/ptr" - - "github.com/percona/percona-server-mysql-operator/pkg/platform" - "github.com/percona/percona-server-mysql-operator/pkg/version" + "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/yaml" ) +func TestPorts(t *testing.T) { + defaultPorts := func() []corev1.ServicePort { + return []corev1.ServicePort{ + { + Name: "http", + Port: 8443, + }, + { + Name: "rw-default", + Port: 3306, + TargetPort: intstr.IntOrString{ + IntVal: 6446, + }, + }, + { + Name: "read-write", + Port: 6446, + }, + { + Name: "read-only", + Port: 6447, + }, + { + Name: "x-read-write", + Port: 6448, + }, + { + Name: "x-read-only", + Port: 6449, + }, + { + Name: "x-default", + Port: 33060, + }, + { + Name: "rw-admin", + Port: 33062, + }, + } + } + + tests := []struct { + name string + specifiedPorts []corev1.ServicePort + expectedPorts []corev1.ServicePort + }{ + { + name: "default ports", + expectedPorts: defaultPorts(), + }, + { + name: "additional ports", + specifiedPorts: []corev1.ServicePort{ + { + Name: "additional port", + Port: 4308, + }, + { + Name: "additional port with target port", + Port: 1337, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 20, + }, + }, + }, + expectedPorts: updateObject(defaultPorts(), func(ports []corev1.ServicePort) []corev1.ServicePort { + ports = append(ports, []corev1.ServicePort{ + { + Name: "additional port", + Port: 4308, + }, + { + Name: "additional port with target port", + Port: 1337, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 20, + }, + }, + }...) + return ports + }), + }, + { + name: "modified ports with additional ports", + specifiedPorts: []corev1.ServicePort{ + { + Name: "http", + Port: 5555, + }, + { + Name: "rw-default", + Port: 6666, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 30, + }, + }, + { + Name: "additional port", + Port: 4308, + }, + { + Name: "additional port with target port", + Port: 1337, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 20, + }, + }, + }, + expectedPorts: updateObject(defaultPorts(), func(ports []corev1.ServicePort) []corev1.ServicePort { + for i, v := range ports { + if v.Name == "http" { + ports[i].Port = 5555 + continue + } + if v.Name == "rw-default" { + ports[i].Port = 6666 + ports[i].TargetPort = intstr.IntOrString{ + Type: intstr.Int, + IntVal: 30, + } + continue + } + } + ports = append(ports, []corev1.ServicePort{ + { + Name: "additional port", + Port: 4308, + }, + { + Name: "additional port with target port", + Port: 1337, + TargetPort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 20, + }, + }, + }...) + return ports + }), + }, + { + name: "modified port with default targetPort", + specifiedPorts: []corev1.ServicePort{ + { + Name: "rw-default", + Port: 6666, + }, + }, + expectedPorts: updateObject(defaultPorts(), func(ports []corev1.ServicePort) []corev1.ServicePort { + for i, v := range ports { + if v.Name == "rw-default" { + ports[i].Port = 6666 + break + } + } + return ports + }), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := ports(tt.specifiedPorts) + if !reflect.DeepEqual(got, tt.expectedPorts) { + gotBytes, err := yaml.Marshal(got) + if err != nil { + t.Fatalf("error marshaling got: %v", err) + } + wantBytes, err := yaml.Marshal(tt.expectedPorts) + if err != nil { + t.Fatalf("error marshaling want: %v", err) + } + t.Fatal(cmp.Diff(string(wantBytes), string(gotBytes))) + } + }) + } +} + +func updateObject[T any](obj T, updateFuncs ...func(obj T) T) T { + for _, f := range updateFuncs { + obj = f(obj) + } + return obj +} + func TestDeployment(t *testing.T) { const ns = "router-ns"