Merge pull request #345 from Till0196/support-nftables

Add nftables support for egress NAT functionality

Author: chez-shanpu

.github/workflows/ci.yaml

@@ -48,6 +48,7 @@ jobs:
         ipv4: ["false", "true"]
         ipv6: ["false", "true"]
         ipv6-primary: ["false", "true"]
+        backend: ["iptables", "nftables"]
         exclude:
         - ipv4: "false"
           ipv6: "false"
@@ -57,6 +58,8 @@ jobs:
         - ipv4: "true"
           ipv6: "false"
           ipv6-primary: "true"
+        - with-ipam: "true"
+          backend: "nftables"
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4
@@ -85,13 +88,22 @@ jobs:
       if: matrix.with-ipam == 'true'
       working-directory: v2/e2e
     - run: make install-coil-egress-v4
-      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'true' && matrix.ipv6 == 'false'
+      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'true' && matrix.ipv6 == 'false' && matrix.backend == 'iptables'
+      working-directory: v2/e2e
+    - run: make install-coil-egress-v4-nft
+      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'true' && matrix.ipv6 == 'false' && matrix.backend == 'nftables'
       working-directory: v2/e2e
     - run: make install-coil-egress-v6
-      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'false' && matrix.ipv6 == 'true'
+      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'false' && matrix.ipv6 == 'true' && matrix.backend == 'iptables'
+      working-directory: v2/e2e
+    - run: make install-coil-egress-v6-nft
+      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'false' && matrix.ipv6 == 'true' && matrix.backend == 'nftables'
       working-directory: v2/e2e
     - run: make install-coil-egress-dualstack
-      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'true' && matrix.ipv6 == 'true'
+      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'true' && matrix.ipv6 == 'true' && matrix.backend == 'iptables'
+      working-directory: v2/e2e
+    - run: make install-coil-egress-dualstack-nft
+      if: matrix.with-ipam == 'false' && matrix.ipv4 == 'true' && matrix.ipv6 == 'true' && matrix.backend == 'nftables'
       working-directory: v2/e2e
     - run: make test TEST_IPAM=${{ matrix.with-ipam }} TEST_EGRESS=true TEST_IPV4=${{ matrix.ipv4 }} TEST_IPV6=${{ matrix.ipv6 }}
       working-directory: v2/e2e
@@ -101,7 +113,7 @@ jobs:
     - uses: actions/upload-artifact@v4
       if: always()
       with:
-        name: logs-ipv4-${{ matrix.ipv4 }}-ipv6-${{ matrix.ipv6 }}-with-ipam-${{ matrix.with-ipam }}-ipv6-primary-${{ matrix.ipv6-primary }}-${{ matrix.kindest-node }}.tar.gz
+        name: logs-ipv4-${{ matrix.ipv4 }}-ipv6-${{ matrix.ipv6 }}-with-ipam-${{ matrix.with-ipam }}-ipv6-primary-${{ matrix.ipv6-primary }}-backend-${{ matrix.backend }}-${{ matrix.kindest-node }}.tar.gz
         path: v2/e2e/logs.tar.gz
   certs-generation:
     name: Cert generation test

v2/Dockerfile

@@ -13,7 +13,7 @@ FROM --platform=$TARGETPLATFORM ghcr.io/cybozu/ubuntu:24.04
 LABEL org.opencontainers.image.source https://github.com/cybozu-go/coil
 
 RUN apt-get update \
-    && apt-get install -y --no-install-recommends netbase kmod iptables iproute2 conntrack \
+    && apt-get install -y --no-install-recommends netbase kmod iptables nftables iproute2 conntrack \
     && rm -rf /var/lib/apt/lists/*
 
 COPY --from=build-env /workdir/work /usr/local/coil

v2/cmd/coil-egress-controller/sub/root.go

@@ -18,6 +18,7 @@ var config struct {
 	webhookAddr string
 	certDir     string
 	egressPort  int32
+	backend     string
 	zapOpts     zap.Options
 
 	enableCertRotation         bool
@@ -29,6 +30,13 @@ var rootCmd = &cobra.Command{
 	Short:   "controller for coil egress related custom resources",
 	Long:    `coil-egress-controller is a Kubernetes controller for coil egress related custom resources.`,
 	Version: v2.Version(),
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		if config.backend != constants.BackendIPTables && config.backend != constants.BackendNFTables {
+			return fmt.Errorf("invalid backend: %s (must be either %s or %s)",
+				config.backend, constants.BackendIPTables, constants.BackendNFTables)
+		}
+		return nil
+	},
 	RunE: func(cmd *cobra.Command, _ []string) error {
 		cmd.SilenceUsage = true
 		return subMain()
@@ -51,6 +59,7 @@ func init() {
 	pf.StringVar(&config.webhookAddr, "webhook-addr", ":9444", "bind address of admission webhook")
 	pf.StringVar(&config.certDir, "cert-dir", "/certs", "directory to locate TLS certs for webhook")
 	pf.Int32Var(&config.egressPort, "egress-port", 5555, "UDP port number used by coil-egress")
+	pf.StringVar(&config.backend, "backend", constants.DefaultBackend, "Backend for egress NAT rules: iptables or nftables (default: iptables)")
 	pf.BoolVar(&config.enableCertRotation, "enable-cert-rotation", constants.DefaultEnableCertRotation, "enables webhook's certificate generation")
 	pf.BoolVar(&config.enableRestartOnCertRefresh, "enable-restart-on-cert-refresh", constants.DefaultEnableRestartOnCertRefresh, "enables pod's restart on webhook certificate refresh")
 

v2/cmd/coil-egress-controller/sub/run.go

@@ -121,11 +121,18 @@ func setupManager(mgr ctrl.Manager, certCompleted chan struct{}) error {
 	if err != nil {
 		return err
 	}
+
+	backend := config.backend
+	if backend == "" {
+		backend = constants.DefaultBackend
+	}
+
 	egressctrl := controllers.EgressReconciler{
-		Client: mgr.GetClient(),
-		Scheme: scheme,
-		Image:  img,
-		Port:   config.egressPort,
+		Client:  mgr.GetClient(),
+		Scheme:  scheme,
+		Image:   img,
+		Port:    config.egressPort,
+		Backend: backend,
 	}
 	if err := egressctrl.SetupWithManager(mgr); err != nil {
 		return err

v2/cmd/coil-egress/sub/root.go

@@ -6,6 +6,7 @@ import (
 	"os"
 
 	v2 "github.com/cybozu-go/coil/v2"
+	"github.com/cybozu-go/coil/v2/pkg/constants"
 	"github.com/spf13/cobra"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/controller-runtime/pkg/log/zap"
@@ -16,6 +17,7 @@ var config struct {
 	healthAddr      string
 	port            int
 	enableSportAuto bool
+	backend         string
 	zapOpts         zap.Options
 }
 
@@ -24,6 +26,13 @@ var rootCmd = &cobra.Command{
 	Short:   "manage foo-over-udp tunnels in egress pods",
 	Long:    `coil-egress manages Foo-over-UDP tunnels in pods created by Egress.`,
 	Version: v2.Version(),
+	PreRunE: func(cmd *cobra.Command, args []string) error {
+		if config.backend != constants.BackendIPTables && config.backend != constants.BackendNFTables {
+			return fmt.Errorf("invalid backend: %s (must be either %s or %s)",
+				config.backend, constants.BackendIPTables, constants.BackendNFTables)
+		}
+		return nil
+	},
 	RunE: func(cmd *cobra.Command, _ []string) error {
 		cmd.SilenceUsage = true
 		return subMain()
@@ -45,6 +54,7 @@ func init() {
 	pf.StringVar(&config.healthAddr, "health-addr", ":8081", "bind address of health/readiness probes")
 	pf.IntVar(&config.port, "fou-port", 5555, "port number for foo-over-udp tunnels")
 	pf.BoolVar(&config.enableSportAuto, "enable-sport-auto", false, "enable automatic source port assignment")
+	pf.StringVar(&config.backend, "backend", constants.DefaultBackend, "Backend for egress NAT rules: iptables or nftables (default: iptables)")
 
 	goflags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(goflags)

v2/cmd/coil-egress/sub/run.go

@@ -112,8 +112,8 @@ func subMain() error {
 		return err
 	}
 
-	setupLog.Info("initialize Egress", "ipv4", ipv4.String(), "ipv6", ipv6.String())
-	eg := founat.NewEgress("eth0", ipv4, ipv6)
+	setupLog.Info("initialize Egress", "ipv4", ipv4.String(), "ipv6", ipv6.String(), "backend", config.backend)
+	eg := founat.NewEgress("eth0", ipv4, ipv6, config.backend)
 	if err := eg.Init(); err != nil {
 		return err
 	}

v2/config/default/egress/v4-nft/kustomization.yaml

@@ -0,0 +1,21 @@
+resources:
+- ../../../crd/egress
+- ../../../rbac/egress
+- ../../../pod/egress/v4-nft
+- ../../../webhook/egress
+
+# [CERTS] Following lines should be commented if automatic cert generation is used.
+patchesStrategicMerge:
+- ../webhook_manifests_patch.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+secretGenerator:
+# [EGRESS] Following lines be uncommented to enable Egress NAT features.
+- name: coilv2-egress-webhook-server-cert
+  files:
+  - ca.crt=../../cert.pem
+  - tls.crt=../../egress-cert.pem
+  - tls.key=../../egress-key.pem
+  type: "kubernetes.io/tls"

v2/config/default/egress/v6-nft/kustomization.yaml

@@ -0,0 +1,21 @@
+resources:
+- ../../../crd/egress
+- ../../../rbac/egress
+- ../../../pod/egress/v6-nft
+- ../../../webhook/egress
+
+# [CERTS] Following lines should be commented if automatic cert generation is used.
+patchesStrategicMerge:
+- ../webhook_manifests_patch.yaml
+
+generatorOptions:
+  disableNameSuffixHash: true
+
+secretGenerator:
+# [EGRESS] Following lines be uncommented to enable Egress NAT features.
+- name: coilv2-egress-webhook-server-cert
+  files:
+  - ca.crt=../../cert.pem
+  - tls.crt=../../egress-cert.pem
+  - tls.key=../../egress-key.pem
+  type: "kubernetes.io/tls"

v2/config/pod/coil-egress-controller-nft-patch.yaml

@@ -0,0 +1,13 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: coil-egress-controller
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: coil-egress-controller
+        args:
+          - --zap-stacktrace-level=panic
+          - --backend=nftables

v2/config/pod/egress/v4-nft/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+# [EGRESS] Following line should be uncommented to enable Egress NAT features.
+- ../../coil-egress-controller.yaml
+- ../v4/coild.yaml
+
+patchesStrategicMerge:
+- ../../coil-egress-controller-nft-patch.yaml