Option for using egress only for originating connections.

Signed-off-by: Patryk Strusiewicz-Surmacki <patryk.pawel.strusiewicz-surmacki@external.telekom.de>

Author: Patryk Strusiewicz-Surmacki

docs/usage.md

@@ -6,23 +6,25 @@ Coil is a Kubernetes-native application and can be controlled with `kubectl`.
 
 For installation, read [setup.md](setup.md).
 
-- [Admin role](#admin-role)
-- [Address pools](#address-pools)
-  - [AddressPool custom resource](#addresspool-custom-resource)
-  - [The default pool](#the-default-pool)
-  - [Using non-default pools](#using-non-default-pools)
-  - [Adding addresses to a pool](#adding-addresses-to-a-pool)
-- [Address blocks](#address-blocks)
-  - [Importing address blocks as routes](#importing-address-blocks-as-routes)
-- [Egress NAT](#egress-nat)
-  - [How it works](#how-it-works)
-  - [Egress custom resource](#egress-custom-resource)
-  - [Client Pods](#client-pods)
-  - [Use NetworkPolicy to prohibit NAT usage](#use-networkpolicy-to-prohibit-nat-usage)
-  - [Session affinity](#session-affinity)
-- [Metrics](#metrics)
-  - [How to scrape metrics](#how-to-scrape-metrics)
-  - [Dashboards](#dashboards)
+- [User manual](#user-manual)
+  - [Admin role](#admin-role)
+  - [Address pools](#address-pools)
+    - [AddressPool custom resource](#addresspool-custom-resource)
+    - [The default pool](#the-default-pool)
+    - [Using non-default pools](#using-non-default-pools)
+    - [Adding addresses to a pool](#adding-addresses-to-a-pool)
+  - [Address blocks](#address-blocks)
+    - [Importing address blocks as routes](#importing-address-blocks-as-routes)
+  - [Egress NAT](#egress-nat)
+    - [How it works](#how-it-works)
+    - [Egress custom resource](#egress-custom-resource)
+    - [Client Pods](#client-pods)
+    - [Use NetworkPolicy to prohibit NAT usage](#use-networkpolicy-to-prohibit-nat-usage)
+    - [Session affinity](#session-affinity)
+    - [Use egress only for connections originating on the client](#use-egress-only-for-connections-originating-on-the-client)
+  - [Metrics](#metrics)
+    - [How to scrape metrics](#how-to-scrape-metrics)
+    - [Dashboards](#dashboards)
 
 ## Admin role
 
@@ -201,6 +203,7 @@ spec:
   destinations:
   - 172.20.0.0/16
   - fd04::/64
+  originatingOnly: true
   replicas: 3
   strategy:
     type: RollingUpdate
@@ -242,6 +245,7 @@ You may customize the container of egress Pods as shown in the above example.
 | Field                   | Type                      | Description                                                          |
 | ----------------------- | ------------------------- | -------------------------------------------------------------------- |
 | `destinations`          | `[]string`                | IP subnets where the packets are SNATed and sent.                    |
+| `originatingOnly`       | `bool`                    | If true, only connections originating in the pod will use egress.    |
 | `replicas`              | `int`                     | Copied to Deployment's `spec.replicas`.  Default is 1.               |
 | `strategy`              | [DeploymentStrategy][]    | Copied to Deployment's `spec.strategy`.                              |
 | `template`              | [PodTemplateSpec][]       | Copied to Deployment's `spec.template`.                              |
@@ -312,6 +316,17 @@ spec:
 
 The default timeout seconds is 10800 (= 3 hours).
 
+### Use egress only for connections originating on the client
+
+If `originatingOnly` is set `true` in the egress definition, only connections originating on the client 
+or incoming onto `fou` interface will use egress FOU interface to send data. 
+In case of incomming connections, the same interface will be used for egress traffic - 
+e.g. if connection will be estabilished on `eth0`, the traffic will not be routed through `fou`,
+but will be handled by `eth0`.
+
+Please be aware that, in case of multiple `egress` resources attached to the client pod, if at least one
+`egress` will have `originatingOnly: true` set, all the other egresses will inherit this behavior.
+
 ## Metrics
 
 Coil exposes two types of Prometheus metrics.

v2/Makefile

@@ -32,6 +32,7 @@ PODNSLIST = pod1 pod2 pod3 pod4 pod5 pod6
 NATNSLIST = nat-client nat-router nat-egress nat-target
 OTHERNSLIST = test-egress-dual test-egress-v4 test-egress-v6 \
 	test-client-dual test-client-v4 test-client-v6 test-client-custom \
+	test-client-dual-oo test-client-v4-oo test-client-v6-oo test-client-custom-oo \
 	test-fou-dual test-fou-v4 test-fou-v6
 WGET_OPTIONS := --retry-on-http-error=503 --retry-connrefused --no-verbose
 WGET := wget $(WGET_OPTIONS)

v2/api/v2/egress_types.go

@@ -67,6 +67,12 @@ type EgressSpec struct {
 	// PodDisruptionBudget is an optional PodDisruptionBudget for Egress NAT pods.
 	// +optional
 	PodDisruptionBudget *EgressPDBSpec `json:"podDisruptionBudget,omitempty"`
+
+	// OriginatingOnly indicates connections that should use egress interface.
+	// If set to true, only outgoing connections that originate in the egress client will use egress.
+	// The default is false.
+	// +optional
+	OriginatingOnly bool `json:"originatingOnly,omitempty"`
 }
 
 // EgressPodTemplate defines pod template for Egress

v2/config/crd/bases/coil.cybozu.com_egresses.yaml

@@ -52,6 +52,12 @@ spec:
                     If set to true, the kernel picks a flow based on the flow hash of the encapsulated packet.
                     The default is false.
                   type: boolean
+                originatingOnly:
+                  description: |-
+                    OriginatingOnly indicates connections that should use egress interface.
+                    If set to true, only outgoing connections that originate in the egress client will use egress.
+                    The default is false.
+                  type: boolean
                 podDisruptionBudget:
                   description: PodDisruptionBudget is an optional PodDisruptionBudget for Egress NAT pods.
                   properties:

v2/config/crd/egress/kustomization.yaml

@@ -3,6 +3,7 @@
 # It should be run by config/default
 resources:
 # [EGRESS] Following files should be uncommented to enable Egress NAT features.
+- ../bases/coil.cybozu.com_addresspools.yaml
 - ../bases/coil.cybozu.com_egresses.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 

v2/controllers/egress_watcher.go

@@ -136,9 +136,10 @@ func (r *EgressWatcher) reconcileEgressClient(ctx context.Context, eg *coilv2.Eg
 }
 
 type gwNets struct {
-	gateway   net.IP
-	networks  []*net.IPNet
-	sportAuto bool
+	gateway         net.IP
+	networks        []*net.IPNet
+	sportAuto       bool
+	originatingOnly bool
 }
 
 func (r *EgressWatcher) getHooks(ctx context.Context, eg *coilv2.Egress, logger *logr.Logger) ([]nodenet.SetupHook, error) {
@@ -203,7 +204,7 @@ func (r *EgressWatcher) hook(gwn gwNets, log *logr.Logger) func(ipv4, ipv6 net.I
 		if err != nil {
 			return err
 		}
-		if err := cl.AddEgress(link, gwn.networks); err != nil {
+		if err := cl.AddEgress(link, gwn.networks, gwn.originatingOnly); err != nil {
 			return err
 		}
 

v2/e2e/Makefile

@@ -113,15 +113,26 @@ setup-nodes:
 TEST_IPAM ?= true
 TEST_EGRESS ?= true
 .PHONY: test
-test: setup-echotest
+test: setup-echotest setup-echotest-image
 	TEST_IPAM=$(TEST_IPAM) TEST_EGRESS=$(TEST_EGRESS) go test -count 1 -v . -args -ginkgo.progress -ginkgo.v
 
-.PHONY: setup-echotest
-setup-echotest:
+.PHONY: build-echotest
+build-echotest:
 	CGO_ENABLED=0 GOARCH=$(ARCH) GOOS=linux go build -o echotest ./echo-server
+
+.PHONY: setup-echotest
+setup-echotest: build-echotest
 	docker cp echotest coil-control-plane:/usr/local/bin
 	rm echotest
 
+.PHONY: build-echotest-image
+build-echotest-image:
+	TARGETARCH=$(ARCH) docker build -t echotest:dev -f echo-server/Dockerfile echo-server/.
+
+.PHONY: setup-echotest-image
+setup-echotest-image: build-echotest-image
+	$(KIND) load docker-image echotest:dev --name coil
+
 .PHONY: logs
 logs:
 	rm -rf logs.tar.gz logs