Merge pull request #1222 from M4t7e/kube-proxy-free-cilium

Introducing Cilium 1.15.1 and Full kube-proxy Replacement

Author: mysticaltech

README.md

@@ -170,6 +170,8 @@ The default is Flannel, but you can also choose Calico or Cilium, by setting the
 
 As Cilium has a lot of interesting and powerful config possibilities, we give you the ability to configure Cilium with the helm `cilium_values` variable (see the cilium specific [helm values](https://github.com/cilium/cilium/blob/master/install/kubernetes/cilium/values.yaml)) before you deploy your cluster.
 
+Cilium supports full kube-proxy replacement. Cilium runs by default in hybrid kube-proxy replacement mode. To achieve a completely kube-proxy-free cluster, set `disable_kube_proxy = true`.
+
 ## Scaling Nodes
 
 Two things can be scaled: the number of nodepools or the number of nodes in these nodepools.

control_planes.tf

@@ -91,6 +91,7 @@ locals {
       module.control_planes[keys(module.control_planes)[0]].private_ipv4_address}:6443"
       token                       = local.k3s_token
       disable-cloud-controller    = true
+      disable-kube-proxy          = var.disable_kube_proxy
       disable                     = local.disable_extras
       kubelet-arg                 = concat(local.kubelet_arg, var.k3s_global_kubelet_args, var.k3s_control_plane_kubelet_args, v.kubelet_args)
       kube-controller-manager-arg = local.kube_controller_manager_arg

init.tf

@@ -37,6 +37,7 @@ resource "null_resource" "first_control_plane" {
           token                       = local.k3s_token
           cluster-init                = true
           disable-cloud-controller    = true
+          disable-kube-proxy          = var.disable_kube_proxy
           disable                     = local.disable_extras
           kubelet-arg                 = local.kubelet_arg
           kube-controller-manager-arg = local.kube_controller_manager_arg

kube.tf.example

@@ -656,6 +656,10 @@ module "kube-hetzner" {
   # Please note that if you are getting 403s from Github, it's also useful to set the version manually. However there is rarely a need for that!
   # calico_version = "v3.25.0"
 
+  # If you want to disable the k3s kube-proxy, use this flag. The default is "false".
+  # Ensure that your CNI is capable of handling all the functionalities typically covered by kube-proxy.
+  # disable_kube_proxy = true
+
   # If you want to disable the k3s default network policy controller, use this flag!
   # Both Calico and Ciliun cni_plugin values override this value to true automatically, the default is "false".
   # disable_network_policy = true

locals.tf

@@ -387,11 +387,23 @@ k8s:
 
 # Replace kube-proxy with Cilium
 kubeProxyReplacement: true
+%{if var.disable_kube_proxy}
+# Enable health check server (healthz) for the kube-proxy replacement
+kubeProxyReplacementHealthzBindAddr: "0.0.0.0:10256"
+%{endif~}
+
+# Access to Kube API Server (mandatory if kube-proxy is disabled)
+k8sServiceHost: "127.0.0.1"
+k8sServicePort: "6444"
 
 # Set Tunnel Mode or Native Routing Mode (supported by Hetzner CCM Route Controller)
 routingMode: "${var.cilium_routing_mode}"
 %{if var.cilium_routing_mode == "native"~}
+# Set the native routable CIDR
 ipv4NativeRoutingCIDR: "${local.cilium_ipv4_native_routing_cidr}"
+
+# Bypass iptables Connection Tracking for Pod traffic (only works in Native Routing Mode)
+installNoConntrackIptablesRules: true
 %{endif~}
 
 endpointRoutes:

variables.tf

@@ -527,6 +527,12 @@ variable "placement_group_disable" {
   description = "Whether to disable placement groups."
 }
 
+variable "disable_kube_proxy" {
+  type        = bool
+  default     = false
+  description = "Disable kube-proxy in K3s (default false)."
+}
+
 variable "disable_network_policy" {
   type        = bool
   default     = false
@@ -575,7 +581,7 @@ variable "cilium_values" {
 
 variable "cilium_version" {
   type        = string
-  default     = "1.14.4"
+  default     = "1.15.1"
   description = "Version of Cilium."
 }