Skip to content

Commit 2edd9b8

Browse files
l-technicorel-technicore
committed
Support for Proxy Protocol Version 2 (PPv2) on NLB via CCM
New E2E - should be possible to create Service type:LoadBalancer and mutate the Proxy Protocol Version on its Listeners
1 parent f3e86d2 commit 2edd9b8

File tree

8 files changed

+243
-8
lines changed

8 files changed

+243
-8
lines changed

‎pkg/cloudprovider/providers/oci/load_balancer_spec.go

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
v1 "k8s.io/api/core/v1"
2727
"k8s.io/apimachinery/pkg/util/sets"
2828
apiservice "k8s.io/kubernetes/pkg/api/v1/service"
29+
"k8s.io/utils/pointer"
2930

3031
"github.com/oracle/oci-cloud-controller-manager/pkg/cloudprovider/providers/oci/config"
3132
"github.com/oracle/oci-cloud-controller-manager/pkg/oci/client"
@@ -234,6 +235,9 @@ const (
234235
// ServiceAnnotationNetworkLoadBalancerIsPreserveSource is a service annotation to enable/disable preserving source information
235236
// on the NLB traffic. Default value when no annotation is given is to enable this for NLBs with externalTrafficPolicy=Local.
236237
ServiceAnnotationNetworkLoadBalancerIsPreserveSource = "oci-network-load-balancer.oraclecloud.com/is-preserve-source"
238+
239+
// ServiceAnnotationNetworkLoadBalancerIsPpv2Enabled is a service annotation to enable/disable PPv2 feature for the listeners of this NLB.
240+
ServiceAnnotationNetworkLoadBalancerIsPpv2Enabled = "oci-network-load-balancer.oraclecloud.com/is-ppv2-enabled"
237241
)
238242

239243
// certificateData is a structure containing the data about a K8S secret required
@@ -1047,13 +1051,22 @@ func getListenersNetworkLoadBalancer(svc *v1.Service, listenerBackendIpVersion [
10471051
listeners := make(map[string]client.GenericListener)
10481052
portsMap := make(map[int][]string)
10491053
mixedProtocolsPortSet := make(map[int]bool)
1054+
var enablePpv2 *bool
10501055

10511056
requireIPv4, requireIPv6 := getRequireIpVersions(listenerBackendIpVersion)
10521057

10531058
for _, servicePort := range svc.Spec.Ports {
10541059
portsMap[int(servicePort.Port)] = append(portsMap[int(servicePort.Port)], string(servicePort.Protocol))
10551060
}
10561061

1062+
if ppv2EnabledValue, ppv2AnnotationSet := svc.Annotations[ServiceAnnotationNetworkLoadBalancerIsPpv2Enabled]; ppv2AnnotationSet {
1063+
if strings.ToLower(ppv2EnabledValue) == "true" {
1064+
enablePpv2 = pointer.Bool(true)
1065+
} else if strings.ToLower(ppv2EnabledValue) == "false" {
1066+
enablePpv2 = pointer.Bool(false)
1067+
}
1068+
}
1069+
10571070
for _, servicePort := range svc.Spec.Ports {
10581071
protocol := string(servicePort.Protocol)
10591072

@@ -1081,8 +1094,9 @@ func getListenersNetworkLoadBalancer(svc *v1.Service, listenerBackendIpVersion [
10811094
}
10821095

10831096
genericListener := client.GenericListener{
1084-
Protocol: &protocol,
1085-
Port: &port,
1097+
Protocol: &protocol,
1098+
Port: &port,
1099+
IsPpv2Enabled: enablePpv2,
10861100
}
10871101
if requireIPv4 {
10881102
genericListener.Name = common.String(listenerName)

‎pkg/cloudprovider/providers/oci/load_balancer_spec_test.go

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ package oci
1717
import (
1818
"context"
1919
"fmt"
20+
"k8s.io/utils/pointer"
2021
"net/http"
2122
"reflect"
2223
"testing"
@@ -9041,6 +9042,89 @@ func Test_getListenersNetworkLoadBalancer(t *testing.T) {
90419042
},
90429043
err: nil,
90439044
},
9045+
"NLB_with_Ppv2_Enabled": {
9046+
service: &v1.Service{
9047+
Spec: v1.ServiceSpec{
9048+
SessionAffinity: v1.ServiceAffinityNone,
9049+
Ports: []v1.ServicePort{
9050+
{
9051+
Protocol: v1.ProtocolTCP,
9052+
Port: int32(67),
9053+
},
9054+
{
9055+
Protocol: v1.ProtocolUDP,
9056+
Port: int32(68),
9057+
},
9058+
},
9059+
},
9060+
ObjectMeta: metav1.ObjectMeta{
9061+
Annotations: map[string]string{
9062+
ServiceAnnotationLoadBalancerType: "nlb",
9063+
ServiceAnnotationNetworkLoadBalancerIsPpv2Enabled: "true",
9064+
},
9065+
},
9066+
},
9067+
listenerBackendIpVersion: []string{IPv4},
9068+
wantListeners: map[string]client.GenericListener{
9069+
"TCP-67": {
9070+
Name: &testTwoListenerNameOne,
9071+
DefaultBackendSetName: common.String(testTwoBackendSetNameOne),
9072+
Protocol: &testTwoProtocolOne,
9073+
Port: &testTwoPortOne,
9074+
IsPpv2Enabled: pointer.Bool(true),
9075+
},
9076+
"UDP-68": {
9077+
Name: &testTwoListenerNameTwo,
9078+
DefaultBackendSetName: common.String(testTwoBackendSetNameTwo),
9079+
Protocol: &testTwoProtocolTwo,
9080+
Port: &testTwoPortTwo,
9081+
IsPpv2Enabled: pointer.Bool(true),
9082+
},
9083+
},
9084+
err: nil,
9085+
},
9086+
9087+
"NLB_with_Ppv2_Disabled": {
9088+
service: &v1.Service{
9089+
Spec: v1.ServiceSpec{
9090+
SessionAffinity: v1.ServiceAffinityNone,
9091+
Ports: []v1.ServicePort{
9092+
{
9093+
Protocol: v1.ProtocolTCP,
9094+
Port: int32(67),
9095+
},
9096+
{
9097+
Protocol: v1.ProtocolUDP,
9098+
Port: int32(68),
9099+
},
9100+
},
9101+
},
9102+
ObjectMeta: metav1.ObjectMeta{
9103+
Annotations: map[string]string{
9104+
ServiceAnnotationLoadBalancerType: "nlb",
9105+
ServiceAnnotationNetworkLoadBalancerIsPpv2Enabled: "xyz",
9106+
},
9107+
},
9108+
},
9109+
listenerBackendIpVersion: []string{IPv4},
9110+
wantListeners: map[string]client.GenericListener{
9111+
"TCP-67": {
9112+
Name: &testTwoListenerNameOne,
9113+
DefaultBackendSetName: common.String(testTwoBackendSetNameOne),
9114+
Protocol: &testTwoProtocolOne,
9115+
Port: &testTwoPortOne,
9116+
IsPpv2Enabled: pointer.Bool(true),
9117+
},
9118+
"UDP-68": {
9119+
Name: &testTwoListenerNameTwo,
9120+
DefaultBackendSetName: common.String(testTwoBackendSetNameTwo),
9121+
Protocol: &testTwoProtocolTwo,
9122+
Port: &testTwoPortTwo,
9123+
IsPpv2Enabled: pointer.Bool(false),
9124+
},
9125+
},
9126+
err: nil,
9127+
},
90449128
}
90459129
for name, tc := range testCases {
90469130
t.Run(name, func(t *testing.T) {

‎pkg/cloudprovider/providers/oci/load_balancer_util.go

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -448,6 +448,9 @@ func hasListenerChanged(logger *zap.SugaredLogger, actual client.GenericListener
448448
if toString(actual.Protocol) != toString(desired.Protocol) {
449449
listenerChanges = append(listenerChanges, fmt.Sprintf(changeFmtStr, "Listener:Protocol", toString(actual.Protocol), toString(desired.Protocol)))
450450
}
451+
if toBool(actual.IsPpv2Enabled) != toBool(desired.IsPpv2Enabled) {
452+
listenerChanges = append(listenerChanges, fmt.Sprintf(changeFmtStr, "Listener:IsPpv2Enabled", toBool(actual.IsPpv2Enabled), toBool(desired.IsPpv2Enabled)))
453+
}
451454

452455
listenerChanges = append(listenerChanges, getSSLConfigurationChanges(actual.SslConfiguration, desired.SslConfiguration)...)
453456
listenerChanges = append(listenerChanges, getConnectionConfigurationChanges(actual.ConnectionConfiguration, desired.ConnectionConfiguration)...)

‎pkg/cloudprovider/providers/oci/load_balancer_util_test.go

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -879,6 +879,74 @@ func TestGetListenerChanges(t *testing.T) {
879879
},
880880
},
881881
},
882+
{
883+
name: "proxy protocol version switch to true - NLB",
884+
desired: map[string]client.GenericListener{
885+
"TCP-80": client.GenericListener{
886+
Name: common.String("TCP-80"),
887+
DefaultBackendSetName: common.String("TCP-80"),
888+
Protocol: common.String("TCP"),
889+
Port: common.Int(80),
890+
IsPpv2Enabled: common.Bool(true),
891+
},
892+
},
893+
actual: map[string]client.GenericListener{
894+
"TCP-80": client.GenericListener{
895+
Name: common.String("TCP-80"),
896+
DefaultBackendSetName: common.String("TCP-80"),
897+
Protocol: common.String("TCP"),
898+
Port: common.Int(80),
899+
IsPpv2Enabled: common.Bool(false),
900+
},
901+
},
902+
expected: []Action{
903+
&ListenerAction{
904+
name: "TCP-80",
905+
actionType: Update,
906+
Listener: client.GenericListener{
907+
Name: common.String("TCP-80"),
908+
DefaultBackendSetName: common.String("TCP-80"),
909+
Protocol: common.String("TCP"),
910+
Port: common.Int(80),
911+
IsPpv2Enabled: common.Bool(true),
912+
},
913+
},
914+
},
915+
},
916+
{
917+
name: "proxy protocol version change - NLB",
918+
desired: map[string]client.GenericListener{
919+
"TCP-80": client.GenericListener{
920+
Name: common.String("TCP-80"),
921+
DefaultBackendSetName: common.String("TCP-80"),
922+
Protocol: common.String("TCP"),
923+
Port: common.Int(80),
924+
IsPpv2Enabled: common.Bool(true),
925+
},
926+
},
927+
actual: map[string]client.GenericListener{
928+
"TCP-80": client.GenericListener{
929+
Name: common.String("TCP-80"),
930+
DefaultBackendSetName: common.String("TCP-80"),
931+
Protocol: common.String("TCP"),
932+
Port: common.Int(80),
933+
IsPpv2Enabled: nil,
934+
},
935+
},
936+
expected: []Action{
937+
&ListenerAction{
938+
name: "TCP-80",
939+
actionType: Update,
940+
Listener: client.GenericListener{
941+
Name: common.String("TCP-80"),
942+
DefaultBackendSetName: common.String("TCP-80"),
943+
Protocol: common.String("TCP"),
944+
Port: common.Int(80),
945+
IsPpv2Enabled: common.Bool(true),
946+
},
947+
},
948+
},
949+
},
882950
{
883951
name: "ssl config change [legacy listeners]",
884952
desired: map[string]client.GenericListener{

‎pkg/oci/client/generic_load_balancer_types.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ type GenericListener struct {
8484
RoutingPolicyName *string
8585
RuleSetNames []string
8686
IpVersion *GenericIpVersion
87+
IsPpv2Enabled *bool
8788
}
8889

8990
type GenericConnectionConfiguration struct {

‎pkg/oci/client/network_load_balancer.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,7 @@ func (c *networkLoadbalancer) CreateListener(ctx context.Context, lbID string, n
304304
DefaultBackendSetName: details.DefaultBackendSetName,
305305
Port: details.Port,
306306
Protocol: networkloadbalancer.ListenerProtocolsEnum(*details.Protocol),
307+
IsPpv2Enabled: details.IsPpv2Enabled,
307308
}
308309

309310
if details.IpVersion != nil {
@@ -338,6 +339,7 @@ func (c *networkLoadbalancer) UpdateListener(ctx context.Context, lbID string, n
338339
DefaultBackendSetName: details.DefaultBackendSetName,
339340
Port: details.Port,
340341
Protocol: networkloadbalancer.ListenerProtocolsEnum(*details.Protocol),
342+
IsPpv2Enabled: details.IsPpv2Enabled,
341343
}
342344

343345
if details.IpVersion != nil {
@@ -569,6 +571,7 @@ func (c *networkLoadbalancer) listenersToGenericListenerDetails(details map[stri
569571
Port: v.Port,
570572
Protocol: &protocol,
571573
IpVersion: &ipVersion,
574+
IsPpv2Enabled: v.IsPpv2Enabled,
572575
}
573576
}
574577
return genericListenerDetails
@@ -659,6 +662,7 @@ func (c *networkLoadbalancer) genericListenerDetailsToListenerDetails(details ma
659662
DefaultBackendSetName: v.DefaultBackendSetName,
660663
Port: v.Port,
661664
Protocol: networkloadbalancer.ListenerProtocolsEnum(*v.Protocol),
665+
IsPpv2Enabled: v.IsPpv2Enabled,
662666
}
663667
if v.IpVersion != nil {
664668
switch *v.IpVersion {

‎test/e2e/cloud-provider-oci/load_balancer.go

Lines changed: 33 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1317,7 +1317,7 @@ var _ = Describe("LB Properties", func() {
13171317
baseName := "lb-properties"
13181318
f := sharedfw.NewDefaultFramework(baseName)
13191319

1320-
Context("[cloudprovider][ccm][lb]", func() {
1320+
Context("[cloudprovider][ccm][lb][properties]", func() {
13211321

13221322
healthCheckTestArray := []struct {
13231323
lbType string
@@ -1385,7 +1385,12 @@ var _ = Describe("LB Properties", func() {
13851385
s.Spec.Ports = []v1.ServicePort{{Name: "http", Port: 80, TargetPort: intstr.FromInt(80)},
13861386
{Name: "https", Port: 443, TargetPort: intstr.FromInt(80)}}
13871387
s.ObjectMeta.Annotations = test.CreationAnnotations
1388-
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationLoadBalancerInternal] = "true"
1388+
if test.lbType == "lb" {
1389+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationLoadBalancerInternal] = "true"
1390+
}
1391+
if test.lbType == "nlb" {
1392+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationNetworkLoadBalancerInternal] = "true"
1393+
}
13891394
})
13901395

13911396
svcPort := int(tcpService.Spec.Ports[0].Port)
@@ -1423,6 +1428,12 @@ var _ = Describe("LB Properties", func() {
14231428
By("changing TCP service health check config")
14241429
tcpService = jig.UpdateServiceOrFail(ns, tcpService.Name, func(s *v1.Service) {
14251430
s.ObjectMeta.Annotations = test.UpdatedAnnotations
1431+
if test.lbType == "lb" {
1432+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationLoadBalancerInternal] = "true"
1433+
}
1434+
if test.lbType == "nlb" {
1435+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationNetworkLoadBalancerInternal] = "true"
1436+
}
14261437
})
14271438

14281439
By("waiting upto 5m0s to verify health check config after modification to initial")
@@ -1432,6 +1443,12 @@ var _ = Describe("LB Properties", func() {
14321443
By("changing TCP service health check config - remove annotations")
14331444
tcpService = jig.UpdateServiceOrFail(ns, tcpService.Name, func(s *v1.Service) {
14341445
s.ObjectMeta.Annotations = test.RemovedAnnotations
1446+
if test.lbType == "lb" {
1447+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationLoadBalancerInternal] = "true"
1448+
}
1449+
if test.lbType == "nlb" {
1450+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationNetworkLoadBalancerInternal] = "true"
1451+
}
14351452
})
14361453

14371454
By("waiting upto 5m0s to verify health check config should fall back to default after removing annotations")
@@ -1686,8 +1703,8 @@ var _ = Describe("LB Properties", func() {
16861703
{
16871704
"nlb",
16881705
map[string]string{
1689-
cloudprovider.ServiceAnnotationLoadBalancerInternal: "true",
1690-
cloudprovider.ServiceAnnotationLoadBalancerType: "nlb",
1706+
cloudprovider.ServiceAnnotationNetworkLoadBalancerInternal: "true",
1707+
cloudprovider.ServiceAnnotationLoadBalancerType: "nlb",
16911708
},
16921709
cloudprovider.ServiceAnnotationNetworkLoadBalancerNetworkSecurityGroups,
16931710
},
@@ -1738,7 +1755,6 @@ var _ = Describe("LB Properties", func() {
17381755
s.Spec.Ports = []v1.ServicePort{{Name: "http", Port: 80, TargetPort: intstr.FromInt(80)},
17391756
{Name: "https", Port: 443, TargetPort: intstr.FromInt(80)}}
17401757
s.ObjectMeta.Annotations = test.Annotations
1741-
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationLoadBalancerInternal] = "true"
17421758
})
17431759

17441760
svcPort := int(tcpService.Spec.Ports[0].Port)
@@ -1781,6 +1797,12 @@ var _ = Describe("LB Properties", func() {
17811797
test.Annotations[test.nsgAnnotation] = nsgIds
17821798
tcpService = jig.UpdateServiceOrFail(ns, tcpService.Name, func(s *v1.Service) {
17831799
s.ObjectMeta.Annotations = test.Annotations
1800+
if test.lbtype == "lb" {
1801+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationLoadBalancerInternal] = "true"
1802+
}
1803+
if test.lbtype == "nlb" {
1804+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationNetworkLoadBalancerInternal] = "true"
1805+
}
17841806
})
17851807
err = f.WaitForLoadBalancerNSGChange(loadBalancer, t.resultantNsgIds, test.lbtype)
17861808
sharedfw.ExpectNoError(err)
@@ -1856,7 +1878,12 @@ var _ = Describe("LB Properties", func() {
18561878
s.Spec.Ports = []v1.ServicePort{{Name: "http", Port: 80, TargetPort: intstr.FromInt(80)},
18571879
{Name: "https", Port: 443, TargetPort: intstr.FromInt(80)}}
18581880
s.ObjectMeta.Annotations = test.CreationAnnotations
1859-
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationLoadBalancerInternal] = "true"
1881+
if test.lbType == "lb" {
1882+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationLoadBalancerInternal] = "true"
1883+
}
1884+
if test.lbType == "nlb" {
1885+
s.ObjectMeta.Annotations[cloudprovider.ServiceAnnotationNetworkLoadBalancerInternal] = "true"
1886+
}
18601887
})
18611888

18621889
svcPort := int(tcpService.Spec.Ports[0].Port)

0 commit comments

Comments
 (0)