Skip to content

Commit f96d651

Browse files
authored
Merge pull request #328 from p-strusiewiczsurmacki-mobica/dualstack-egress
DualStack support for egress
2 parents 0648c38 + a922fce commit f96d651

20 files changed

+474
-216
lines changed

.github/workflows/ci.yaml

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,18 @@ jobs:
4545
matrix:
4646
kindest-node: ["1.29.12", "1.30.8", "1.31.4"]
4747
with-ipam: ["false", "true"]
48+
ipv4: ["false", "true"]
4849
ipv6: ["false", "true"]
50+
ipv6-primary: ["false", "true"]
51+
exclude:
52+
- ipv4: "false"
53+
ipv6: "false"
54+
- ipv4: "false"
55+
ipv6: "true"
56+
ipv6-primary: "true"
57+
- ipv4: "true"
58+
ipv6: "false"
59+
ipv6-primary: "true"
4960
runs-on: ubuntu-24.04
5061
steps:
5162
- uses: actions/checkout@v4
@@ -64,30 +75,33 @@ jobs:
6475
sudo systemctl restart docker.service
6576
sleep 10
6677
echo TEST_IPV6=true >> $GITHUB_ENV
67-
- run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }} WITH_KINDNET=false TEST_IPV6=${{ matrix.ipv6 }}
78+
- run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }} WITH_KINDNET=false TEST_IPV4=${{ matrix.ipv4 }} TEST_IPV6=${{ matrix.ipv6 }} IPV6_PRIMARY=${{ matrix.ipv6-primary }}
6879
if: matrix.with-ipam == 'true'
6980
working-directory: v2/e2e
70-
- run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }} WITH_KINDNET=true TEST_IPV6=${{ matrix.ipv6 }}
81+
- run: make start KUBERNETES_VERSION=${{ matrix.kindest-node }} WITH_KINDNET=true TEST_IPV4=${{ matrix.ipv4 }} TEST_IPV6=${{ matrix.ipv6 }} IPV6_PRIMARY=${{ matrix.ipv6-primary }}
7182
if: matrix.with-ipam == 'false'
7283
working-directory: v2/e2e
7384
- run: make install-coil
7485
if: matrix.with-ipam == 'true'
7586
working-directory: v2/e2e
7687
- run: make install-coil-egress-v4
77-
if: matrix.with-ipam == 'false' && matrix.ipv6 == 'false'
88+
if: matrix.with-ipam == 'false' && matrix.ipv4 == 'true' && matrix.ipv6 == 'false'
7889
working-directory: v2/e2e
7990
- run: make install-coil-egress-v6
80-
if: matrix.with-ipam == 'false' && matrix.ipv6 == 'true'
91+
if: matrix.with-ipam == 'false' && matrix.ipv4 == 'false' && matrix.ipv6 == 'true'
8192
working-directory: v2/e2e
82-
- run: make test TEST_IPAM=${{ matrix.with-ipam }} TEST_EGRESS=true TEST_IPV6=${{ matrix.ipv6 }}
93+
- run: make install-coil-egress-dualstack
94+
if: matrix.with-ipam == 'false' && matrix.ipv4 == 'true' && matrix.ipv6 == 'true'
95+
working-directory: v2/e2e
96+
- run: make test TEST_IPAM=${{ matrix.with-ipam }} TEST_EGRESS=true TEST_IPV4=${{ matrix.ipv4 }} TEST_IPV6=${{ matrix.ipv6 }}
8397
working-directory: v2/e2e
8498
- run: make logs
8599
working-directory: v2/e2e
86100
if: always()
87101
- uses: actions/upload-artifact@v4
88102
if: always()
89103
with:
90-
name: logs-ipv6-${{ matrix.ipv6 }}-with-ipam-${{ matrix.with-ipam }}-${{ matrix.kindest-node }}.tar.gz
104+
name: logs-ipv4-${{ matrix.ipv4 }}-ipv6-${{ matrix.ipv6 }}-with-ipam-${{ matrix.with-ipam }}-ipv6-primary-${{ matrix.ipv6-primary }}-${{ matrix.kindest-node }}.tar.gz
91105
path: v2/e2e/logs.tar.gz
92106
certs-generation:
93107
name: Cert generation test

v2/config/default/egress/v4/kustomization.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ generatorOptions:
1414
secretGenerator:
1515
# [EGRESS] Following lines be uncommented to enable Egress NAT features.
1616
- name: coilv2-egress-webhook-server-cert
17-
# [CERTS] Following lines should be commented if automatic cert generation is used.
1817
files:
1918
- ca.crt=../../cert.pem
2019
- tls.crt=../../egress-cert.pem

v2/config/default/egress/v6/kustomization.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ generatorOptions:
1414
secretGenerator:
1515
# [EGRESS] Following lines be uncommented to enable Egress NAT features.
1616
- name: coilv2-egress-webhook-server-cert
17-
# [CERTS] Following lines should be commented if automatic cert generation is used.
1817
files:
1918
- ca.crt=../../cert.pem
2019
- tls.crt=../../egress-cert.pem

v2/controllers/egress_controller.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,8 @@ func (r *EgressReconciler) reconcileService(ctx context.Context, log logr.Logger
360360
eg.Spec.SessionAffinityConfig.DeepCopyInto(sac)
361361
svc.Spec.SessionAffinityConfig = sac
362362
}
363+
svc.Spec.IPFamilyPolicy = new(corev1.IPFamilyPolicy)
364+
*svc.Spec.IPFamilyPolicy = corev1.IPFamilyPolicyPreferDualStack
363365

364366
return nil
365367
})

v2/controllers/egress_watcher.go

Lines changed: 20 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ func (r *EgressWatcher) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.R
109109
}
110110

111111
func (r *EgressWatcher) reconcileEgressClient(ctx context.Context, eg *coilv2.Egress, pod *corev1.Pod, logger *logr.Logger) error {
112-
hook, err := r.getHook(ctx, eg, logger)
112+
hooks, err := r.getHooks(ctx, eg, logger)
113113
if err != nil {
114114
return fmt.Errorf("failed to setup NAT hook: %w", err)
115115
}
@@ -125,8 +125,11 @@ func (r *EgressWatcher) reconcileEgressClient(ctx context.Context, eg *coilv2.Eg
125125
ipv6 = ip.To16()
126126
}
127127
}
128-
if err := r.PodNet.Update(ipv4, ipv6, hook, pod); err != nil {
129-
return fmt.Errorf("failed to update NAT configuration: %w", err)
128+
129+
for _, hook := range hooks {
130+
if err := r.PodNet.Update(ipv4, ipv6, hook, pod); err != nil {
131+
return fmt.Errorf("failed to update NAT configuration: %w", err)
132+
}
130133
}
131134

132135
return nil
@@ -138,49 +141,39 @@ type gwNets struct {
138141
sportAuto bool
139142
}
140143

141-
func (r *EgressWatcher) getHook(ctx context.Context, eg *coilv2.Egress, logger *logr.Logger) (nodenet.SetupHook, error) {
144+
func (r *EgressWatcher) getHooks(ctx context.Context, eg *coilv2.Egress, logger *logr.Logger) ([]nodenet.SetupHook, error) {
142145
var gw gwNets
143146
svc := &corev1.Service{}
144147

145148
if err := r.Get(ctx, client.ObjectKey{Namespace: eg.Namespace, Name: eg.Name}, svc); err != nil {
146149
return nil, err
147150
}
148151

149-
// See getHook in coild_server.go
150-
svcIP := net.ParseIP(svc.Spec.ClusterIP)
151-
if svcIP == nil {
152-
return nil, fmt.Errorf("invalid ClusterIP in Service %s %s", eg.Name, svc.Spec.ClusterIP)
153-
}
154-
var subnets []*net.IPNet
155-
if ip4 := svcIP.To4(); ip4 != nil {
156-
svcIP = ip4
157-
for _, sn := range eg.Spec.Destinations {
158-
_, subnet, err := net.ParseCIDR(sn)
159-
if err != nil {
160-
return nil, fmt.Errorf("invalid network in Egress %s", eg.Name)
161-
}
162-
if subnet.IP.To4() != nil {
163-
subnets = append(subnets, subnet)
164-
}
152+
hooks := []nodenet.SetupHook{}
153+
for _, clusterIP := range svc.Spec.ClusterIPs {
154+
var subnets []*net.IPNet
155+
svcIP := net.ParseIP(clusterIP)
156+
if svcIP == nil {
157+
return nil, fmt.Errorf("invalid ClusterIP in Service %s %s", eg.Name, svc.Spec.ClusterIP)
165158
}
166-
} else {
159+
167160
for _, sn := range eg.Spec.Destinations {
168161
_, subnet, err := net.ParseCIDR(sn)
169162
if err != nil {
170163
return nil, fmt.Errorf("invalid network in Egress %s", eg.Name)
171164
}
172-
if subnet.IP.To4() == nil {
165+
if (svcIP.To4() != nil) == (subnet.IP.To4() != nil) {
173166
subnets = append(subnets, subnet)
174167
}
175168
}
176-
}
177169

178-
if len(subnets) > 0 {
179-
gw = gwNets{gateway: svcIP, networks: subnets, sportAuto: eg.Spec.FouSourcePortAuto}
180-
return r.hook(gw, logger), nil
170+
if len(subnets) > 0 {
171+
gw = gwNets{gateway: svcIP, networks: subnets, sportAuto: eg.Spec.FouSourcePortAuto}
172+
hooks = append(hooks, r.hook(gw, logger))
173+
}
181174
}
182175

183-
return nil, nil
176+
return hooks, nil
184177
}
185178

186179
func (r *EgressWatcher) hook(gwn gwNets, log *logr.Logger) func(ipv4, ipv6 net.IP) error {

v2/e2e/Makefile

Lines changed: 40 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,26 @@ export KUBECTL
1010

1111
KIND_CONFIG = kind-config.yaml
1212
ifeq ($(TEST_IPV6),true)
13-
ifeq ($(WITH_KINDNET),true)
14-
KIND_CONFIG = kind-config_kindnet_v6.yaml
15-
else
16-
KIND_CONFIG = kind-config_v6.yaml
13+
ifeq ($(TEST_IPV4),true)
14+
ifeq ($(IPV6_PRIMARY),true)
15+
ifeq ($(WITH_KINDNET),true)
16+
KIND_CONFIG = kind-config_dualstack_v6_kindnet.yaml
17+
else
18+
KIND_CONFIG = kind-config_dualstack_v6.yaml
19+
endif
20+
else
21+
ifeq ($(WITH_KINDNET),true)
22+
KIND_CONFIG = kind-config_dualstack_kindnet.yaml
23+
else
24+
KIND_CONFIG = kind-config_dualstack.yaml
25+
endif
26+
endif
27+
else
28+
ifeq ($(WITH_KINDNET),true)
29+
KIND_CONFIG = kind-config_kindnet_v6.yaml
30+
else
31+
KIND_CONFIG = kind-config_v6.yaml
32+
endif
1733
endif
1834
else
1935
ifeq ($(WITH_KINDNET),true)
@@ -50,7 +66,7 @@ install-coil-egress-v4: setup-nodes
5066
$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
5167
$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
5268
$(KUBECTL) rollout status daemonset coild -n kube-system --timeout=3m
53-
./kindnet-conf --action set --file 10-coil.conflist
69+
./kindnet-conf --action set --cni-config 10-coil.conflist
5470
rm -rf tmp kindnet-conf
5571

5672
.PHONY: install-coil-egress-v6
@@ -65,7 +81,24 @@ install-coil-egress-v6: setup-nodes
6581
$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
6682
$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
6783
$(KUBECTL) rollout status daemonset coild -n kube-system --timeout=3m
68-
./kindnet-conf --action set --file 10-coil.conflist --protocol v6
84+
./kindnet-conf --action set --cni-config 10-coil.conflist --protocol v6
85+
rm -rf tmp kindnet-conf
86+
87+
.PHONY: install-coil-egress-v6
88+
install-coil-egress-dualstack: setup-nodes
89+
rm -rf tmp
90+
mkdir tmp 2> /dev/null
91+
$(KUBECTL) rollout status daemonset kindnet -n kube-system --timeout 120s
92+
CGO_ENABLED=0 go build -o kindnet-conf ./kindnet-configurer
93+
./kindnet-conf --action get --protocol v4
94+
./kindnet-conf --action get --protocol v6
95+
$(KIND) load docker-image --name coil coil:dev
96+
$(KUSTOMIZE) build --load-restrictor=LoadRestrictionsNone configs/egress/dualstack | $(KUBECTL) apply -f -
97+
$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
98+
$(KUBECTL) -n kube-system wait --timeout=3m --for=condition=available deployment/coil-egress-controller
99+
$(KUBECTL) rollout status daemonset coild -n kube-system --timeout=3m
100+
./kindnet-conf --action set --cni-config 10-coil.conflist --protocol v4
101+
./kindnet-conf --action set --cni-config 10-coil.conflist --protocol v6
69102
rm -rf tmp kindnet-conf
70103

71104
.PHONY: setup-nodes
@@ -105,6 +138,7 @@ enable-certs-rotation:
105138
@sed -i "9,21 {s/^# //}" kustomization.yaml
106139
@sed -i "18,24 {s/^# //}" configs/egress/v4/kustomization.yaml
107140
@sed -i "18,24 {s/^# //}" configs/egress/v6/kustomization.yaml
141+
@sed -i "18,24 {s/^# //}" configs/egress/dualstack/kustomization.yaml
108142
@sed -i -E 's/^(- coil-.*controller_role\.yaml)/# \1/g' ../config/rbac/kustomization.yaml
109143
@sed -i -E 's/^# (- coil-.*controller-certs_role\.yaml)/\1/g' ../config/rbac/kustomization.yaml
110144
@sed -i -E 's/^(- \.\.\/coil-.*controller_role\.yaml)/# \1/g' ../config/rbac/egress/kustomization.yaml

0 commit comments

Comments
 (0)