Skip to content

Commit ede5100

Browse files
l-technicorel-technicore
authored
Merge pull request #385 from jhorwit2/jah/filter-backend-nodes
Add support for filtering the nodes based on label selectors for an LB service by annotation
2 parents b1abe12 + 0a06c83 commit ede5100

File tree

3 files changed

+321
-4
lines changed

3 files changed

+321
-4
lines changed

‎pkg/cloudprovider/providers/oci/load_balancer.go

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -351,13 +351,60 @@ func (clb *CloudLoadBalancerProvider) createLoadBalancer(ctx context.Context, sp
351351

352352
}
353353

354+
// getNodeFilter extracts the node filter based on load balancer type.
355+
// if no selector is defined then an all label selector object is returned to match everything.
356+
func getNodeFilter(svc *v1.Service) (labels.Selector, error) {
357+
lbType := getLoadBalancerType(svc)
358+
359+
var labelSelector string
360+
361+
switch lbType {
362+
case NLB:
363+
labelSelector = svc.Annotations[ServiceAnnotationNetworkLoadBalancerNodeFilter]
364+
default:
365+
labelSelector = svc.Annotations[ServiceAnnotationLoadBalancerNodeFilter]
366+
}
367+
368+
if labelSelector == "" {
369+
return labels.Everything(), nil
370+
}
371+
372+
return labels.Parse(labelSelector)
373+
}
374+
375+
// filterNodes based on the label selector, if present, and returns the set of nodes
376+
// that should be backends in the load balancer.
377+
func filterNodes(svc *v1.Service, nodes []*v1.Node) ([]*v1.Node, error) {
378+
379+
selector, err := getNodeFilter(svc)
380+
if err != nil {
381+
return nil, err
382+
}
383+
384+
var filteredNodes []*v1.Node
385+
for _, n := range nodes {
386+
if selector.Matches(labels.Set(n.GetLabels())) {
387+
filteredNodes = append(filteredNodes, n)
388+
}
389+
}
390+
391+
return filteredNodes, nil
392+
}
393+
354394
// EnsureLoadBalancer creates a new load balancer or updates the existing one.
355395
// Returns the status of the balancer (i.e it's public IP address if one exists).
356396
func (cp *CloudProvider) EnsureLoadBalancer(ctx context.Context, clusterName string, service *v1.Service, nodes []*v1.Node) (*v1.LoadBalancerStatus, error) {
357397
startTime := time.Now()
358398
lbName := GetLoadBalancerName(service)
359399
loadBalancerType := getLoadBalancerType(service)
360400
logger := cp.logger.With("loadBalancerName", lbName, "serviceName", service.Name, "loadBalancerType", loadBalancerType)
401+
402+
nodes, err := filterNodes(service, nodes)
403+
if err != nil {
404+
logger.With(zap.Error(err)).Error("Failed to filter nodes with label selector")
405+
return nil, err
406+
}
407+
361408
logger.With("nodes", len(nodes)).Info("Ensuring load balancer")
362409

363410
dimensionsMap := make(map[string]string)

‎pkg/cloudprovider/providers/oci/load_balancer_spec.go

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,10 @@ import (
3434
)
3535

3636
const (
37-
LB = "lb"
38-
NLB = "nlb"
39-
LBHealthCheckIntervalMin = 1000
40-
LBHealthCheckIntervalMax = 1800000
37+
LB = "lb"
38+
NLB = "nlb"
39+
LBHealthCheckIntervalMin = 1000
40+
LBHealthCheckIntervalMax = 1800000
4141
NLBHealthCheckIntervalMin = 10000
4242
NLBHealthCheckIntervalMax = 1800000
4343
)
@@ -138,6 +138,10 @@ const (
138138

139139
// ServiceAnnotationLoadBalancerType is a service annotation for specifying lb type
140140
ServiceAnnotationLoadBalancerType = "oci.oraclecloud.com/load-balancer-type"
141+
142+
// ServiceAnnotationLoadBalancerNodeFilter is a service annotation to select specific nodes as your backend in the LB
143+
// based on label selector.
144+
ServiceAnnotationLoadBalancerNodeFilter = "oci.oraclecloud.com/node-label-selector"
141145
)
142146

143147
// NLB specific annotations
@@ -181,6 +185,10 @@ const (
181185
// ServiceAnnotationNetworkLoadBalancerInitialFreeformTagsOverride is a service annotation for specifying
182186
// freeform tags on the nlb
183187
ServiceAnnotationNetworkLoadBalancerInitialFreeformTagsOverride = "oci-network-load-balancer.oraclecloud.com/freeform-tags"
188+
189+
// ServiceAnnotationNetworkLoadBalancerNodeFilter is a service annotation to select specific nodes as your backend in the NLB
190+
// based on label selector.
191+
ServiceAnnotationNetworkLoadBalancerNodeFilter = "oci-network-load-balancer.oraclecloud.com/node-label-selector"
184192
)
185193

186194
// certificateData is a structure containing the data about a K8S secret required

‎pkg/cloudprovider/providers/oci/load_balancer_test.go

Lines changed: 262 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,268 @@ import (
3030
"github.com/oracle/oci-go-sdk/v50/core"
3131
)
3232

33+
func newNodeObj(name string, labels map[string]string) *v1.Node {
34+
return &v1.Node{
35+
ObjectMeta: metav1.ObjectMeta{
36+
Name: name,
37+
Labels: labels,
38+
},
39+
}
40+
}
41+
42+
func Test_filterNodes(t *testing.T) {
43+
testCases := map[string]struct {
44+
nodes []*v1.Node
45+
service *v1.Service
46+
expected []*v1.Node
47+
}{
48+
"lb - no annotation": {
49+
nodes: []*v1.Node{
50+
newNodeObj("node1", nil),
51+
newNodeObj("node2", nil),
52+
},
53+
service: &v1.Service{
54+
ObjectMeta: metav1.ObjectMeta{
55+
Namespace: "kube-system",
56+
Name: "testservice",
57+
Annotations: map[string]string{},
58+
},
59+
},
60+
expected: []*v1.Node{
61+
newNodeObj("node1", nil),
62+
newNodeObj("node2", nil),
63+
},
64+
},
65+
"nlb - no annotation": {
66+
nodes: []*v1.Node{
67+
newNodeObj("node1", nil),
68+
newNodeObj("node2", nil),
69+
},
70+
service: &v1.Service{
71+
ObjectMeta: metav1.ObjectMeta{
72+
Namespace: "kube-system",
73+
Name: "testservice",
74+
Annotations: map[string]string{
75+
ServiceAnnotationLoadBalancerType: "nlb",
76+
},
77+
},
78+
},
79+
expected: []*v1.Node{
80+
newNodeObj("node1", nil),
81+
newNodeObj("node2", nil),
82+
},
83+
},
84+
"lb - empty annotation": {
85+
nodes: []*v1.Node{
86+
newNodeObj("node1", nil),
87+
newNodeObj("node2", nil),
88+
},
89+
service: &v1.Service{
90+
ObjectMeta: metav1.ObjectMeta{
91+
Namespace: "kube-system",
92+
Name: "testservice",
93+
Annotations: map[string]string{
94+
ServiceAnnotationLoadBalancerNodeFilter: "",
95+
},
96+
},
97+
},
98+
expected: []*v1.Node{
99+
newNodeObj("node1", nil),
100+
newNodeObj("node2", nil),
101+
},
102+
},
103+
"nlb - empty annotation": {
104+
nodes: []*v1.Node{
105+
newNodeObj("node1", nil),
106+
newNodeObj("node2", nil),
107+
},
108+
service: &v1.Service{
109+
ObjectMeta: metav1.ObjectMeta{
110+
Namespace: "kube-system",
111+
Name: "testservice",
112+
Annotations: map[string]string{
113+
ServiceAnnotationLoadBalancerType: "nlb",
114+
ServiceAnnotationNetworkLoadBalancerNodeFilter: "",
115+
},
116+
},
117+
},
118+
expected: []*v1.Node{
119+
newNodeObj("node1", nil),
120+
newNodeObj("node2", nil),
121+
},
122+
},
123+
"lb - single selector select all": {
124+
nodes: []*v1.Node{
125+
newNodeObj("node1", map[string]string{"foo": "bar"}),
126+
newNodeObj("node2", map[string]string{"foo": "bar"}),
127+
},
128+
service: &v1.Service{
129+
ObjectMeta: metav1.ObjectMeta{
130+
Namespace: "kube-system",
131+
Name: "testservice",
132+
Annotations: map[string]string{
133+
ServiceAnnotationLoadBalancerNodeFilter: "foo=bar",
134+
},
135+
},
136+
},
137+
expected: []*v1.Node{
138+
newNodeObj("node1", map[string]string{"foo": "bar"}),
139+
newNodeObj("node2", map[string]string{"foo": "bar"}),
140+
},
141+
},
142+
"nlb - single selector select all": {
143+
nodes: []*v1.Node{
144+
newNodeObj("node1", map[string]string{"foo": "bar"}),
145+
newNodeObj("node2", map[string]string{"foo": "bar"}),
146+
},
147+
service: &v1.Service{
148+
ObjectMeta: metav1.ObjectMeta{
149+
Namespace: "kube-system",
150+
Name: "testservice",
151+
Annotations: map[string]string{
152+
ServiceAnnotationLoadBalancerType: "nlb",
153+
ServiceAnnotationNetworkLoadBalancerNodeFilter: "foo=bar",
154+
},
155+
},
156+
},
157+
expected: []*v1.Node{
158+
newNodeObj("node1", map[string]string{"foo": "bar"}),
159+
newNodeObj("node2", map[string]string{"foo": "bar"}),
160+
},
161+
},
162+
"lb - single selector select some": {
163+
nodes: []*v1.Node{
164+
newNodeObj("node1", map[string]string{"foo": "bar"}),
165+
newNodeObj("node2", map[string]string{"foo": "baz"}),
166+
},
167+
service: &v1.Service{
168+
ObjectMeta: metav1.ObjectMeta{
169+
Namespace: "kube-system",
170+
Name: "testservice",
171+
Annotations: map[string]string{
172+
ServiceAnnotationLoadBalancerNodeFilter: "foo=bar",
173+
},
174+
},
175+
},
176+
expected: []*v1.Node{
177+
newNodeObj("node1", map[string]string{"foo": "bar"}),
178+
},
179+
},
180+
"nlb - single selector select some": {
181+
nodes: []*v1.Node{
182+
newNodeObj("node1", map[string]string{"foo": "bar"}),
183+
newNodeObj("node2", map[string]string{"foo": "baz"}),
184+
},
185+
service: &v1.Service{
186+
ObjectMeta: metav1.ObjectMeta{
187+
Namespace: "kube-system",
188+
Name: "testservice",
189+
Annotations: map[string]string{
190+
ServiceAnnotationLoadBalancerType: "nlb",
191+
ServiceAnnotationNetworkLoadBalancerNodeFilter: "foo=bar",
192+
},
193+
},
194+
},
195+
expected: []*v1.Node{
196+
newNodeObj("node1", map[string]string{"foo": "bar"}),
197+
},
198+
},
199+
"lb - multi selector select all": {
200+
nodes: []*v1.Node{
201+
newNodeObj("node1", map[string]string{"foo": "bar"}),
202+
newNodeObj("node2", map[string]string{"foo": "baz"}),
203+
},
204+
service: &v1.Service{
205+
ObjectMeta: metav1.ObjectMeta{
206+
Namespace: "kube-system",
207+
Name: "testservice",
208+
Annotations: map[string]string{
209+
ServiceAnnotationLoadBalancerNodeFilter: "foo",
210+
},
211+
},
212+
},
213+
expected: []*v1.Node{
214+
newNodeObj("node1", map[string]string{"foo": "bar"}),
215+
newNodeObj("node2", map[string]string{"foo": "baz"}),
216+
},
217+
},
218+
"nlb - multi selector select all": {
219+
nodes: []*v1.Node{
220+
newNodeObj("node1", map[string]string{"foo": "bar"}),
221+
newNodeObj("node2", map[string]string{"foo": "baz"}),
222+
},
223+
service: &v1.Service{
224+
ObjectMeta: metav1.ObjectMeta{
225+
Namespace: "kube-system",
226+
Name: "testservice",
227+
Annotations: map[string]string{
228+
ServiceAnnotationLoadBalancerType: "nlb",
229+
ServiceAnnotationNetworkLoadBalancerNodeFilter: "foo",
230+
},
231+
},
232+
},
233+
expected: []*v1.Node{
234+
newNodeObj("node1", map[string]string{"foo": "bar"}),
235+
newNodeObj("node2", map[string]string{"foo": "baz"}),
236+
},
237+
},
238+
"lb - multi selector select some": {
239+
nodes: []*v1.Node{
240+
newNodeObj("node1", map[string]string{"foo": "bar"}),
241+
newNodeObj("node2", map[string]string{"foo": "joe"}),
242+
newNodeObj("node2", map[string]string{"foo": "baz"}),
243+
},
244+
service: &v1.Service{
245+
ObjectMeta: metav1.ObjectMeta{
246+
Namespace: "kube-system",
247+
Name: "testservice",
248+
Annotations: map[string]string{
249+
ServiceAnnotationLoadBalancerNodeFilter: "foo in (bar, baz)",
250+
},
251+
},
252+
},
253+
expected: []*v1.Node{
254+
newNodeObj("node1", map[string]string{"foo": "bar"}),
255+
newNodeObj("node2", map[string]string{"foo": "baz"}),
256+
},
257+
},
258+
"nlb - multi selector select some": {
259+
nodes: []*v1.Node{
260+
newNodeObj("node1", map[string]string{"foo": "bar"}),
261+
newNodeObj("node2", map[string]string{"foo": "joe"}),
262+
newNodeObj("node2", map[string]string{"foo": "baz"}),
263+
},
264+
service: &v1.Service{
265+
ObjectMeta: metav1.ObjectMeta{
266+
Namespace: "kube-system",
267+
Name: "testservice",
268+
Annotations: map[string]string{
269+
ServiceAnnotationLoadBalancerType: "nlb",
270+
ServiceAnnotationNetworkLoadBalancerNodeFilter: "foo in (bar, baz)",
271+
},
272+
},
273+
},
274+
expected: []*v1.Node{
275+
newNodeObj("node1", map[string]string{"foo": "bar"}),
276+
newNodeObj("node2", map[string]string{"foo": "baz"}),
277+
},
278+
},
279+
}
280+
281+
for name, tc := range testCases {
282+
t.Run(name, func(t *testing.T) {
283+
nodes, err := filterNodes(tc.service, tc.nodes)
284+
if err != nil {
285+
t.Fatal(err)
286+
}
287+
288+
if !reflect.DeepEqual(nodes, tc.expected) {
289+
t.Errorf("expected: %+v got %+v", tc.expected, nodes)
290+
}
291+
})
292+
}
293+
}
294+
33295
func Test_getDefaultLBSubnets(t *testing.T) {
34296
type args struct {
35297
subnet1 string

0 commit comments

Comments
 (0)