Merge pull request #12101 from tico88612/refactor/cilium-install

Refactor Cilium CNI installation

Author: k8s-ci-robot

README.md

@@ -119,7 +119,7 @@ Note:
 - Network Plugin
   - [cni-plugins](https://github.com/containernetworking/plugins) 1.4.1
   - [calico](https://github.com/projectcalico/calico) 3.29.3
-  - [cilium](https://github.com/cilium/cilium) 1.15.9
+  - [cilium](https://github.com/cilium/cilium) 1.17.3
   - [flannel](https://github.com/flannel-io/flannel) 0.22.0
   - [kube-ovn](https://github.com/alauda/kube-ovn) 1.12.21
   - [kube-router](https://github.com/cloudnativelabs/kube-router) 2.1.1

docs/CNI/cilium.md

@@ -237,7 +237,7 @@ cilium_operator_extra_volume_mounts:
 ## Choose Cilium version
 
 ```yml
-cilium_version: "1.15.9"
+cilium_version: "1.17.3"
 ```
 
 ## Add variable to config

roles/kubespray_defaults/defaults/main/download.yml

@@ -113,7 +113,7 @@ flannel_cni_version: 1.1.2
 weave_version: 2.8.7
 cni_version: "{{ (cni_binary_checksums['amd64'] | dict2items)[0].key }}"
 
-cilium_version: "1.15.9"
+cilium_version: "1.17.3"
 cilium_cli_version: "{{ (ciliumcli_binary_checksums['amd64'] | dict2items)[0].key }}"
 cilium_enable_hubble: false
 
@@ -261,13 +261,13 @@ cilium_operator_image_tag: "v{{ cilium_version }}"
 cilium_hubble_relay_image_repo: "{{ quay_image_repo }}/cilium/hubble-relay"
 cilium_hubble_relay_image_tag: "v{{ cilium_version }}"
 cilium_hubble_certgen_image_repo: "{{ quay_image_repo }}/cilium/certgen"
-cilium_hubble_certgen_image_tag: "v0.1.8"
+cilium_hubble_certgen_image_tag: "v0.2.1"
 cilium_hubble_ui_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui"
-cilium_hubble_ui_image_tag: "v0.11.0"
+cilium_hubble_ui_image_tag: "v0.13.2"
 cilium_hubble_ui_backend_image_repo: "{{ quay_image_repo }}/cilium/hubble-ui-backend"
-cilium_hubble_ui_backend_image_tag: "v0.11.0"
-cilium_hubble_envoy_image_repo: "{{ docker_image_repo }}/envoyproxy/envoy"
-cilium_hubble_envoy_image_tag: "v1.22.5"
+cilium_hubble_ui_backend_image_tag: "v0.13.2"
+cilium_hubble_envoy_image_repo: "{{ quay_image_repo }}/cilium/cilium-envoy"
+cilium_hubble_envoy_image_tag: "v1.32.5-1744305768-f9ddca7dcd91f7ca25a505560e655c47d3dec2cf"
 kube_ovn_container_image_repo: "{{ docker_image_repo }}/kubeovn/kube-ovn"
 kube_ovn_container_image_tag: "v{{ kube_ovn_version }}"
 kube_ovn_vpc_container_image_repo: "{{ docker_image_repo }}/kubeovn/vpc-nat-gateway"

roles/kubespray_defaults/vars/main/checksums.yml

@@ -556,6 +556,11 @@ calicoctl_binary_checksums:
     3.27.0: sha256:3de46d8bc30c6f9d9387d484ed62a5655c1f204b1b831b5a90f0a0d1c1ffd752
 ciliumcli_binary_checksums:
   arm64:
+    0.18.3: sha256:e0588268fc9ab6e0b7a363c4e15ecf69ed2a4cade956ab272745262e456f0e54
+    0.18.2: sha256:db3fae09ba005d6d345858655777bb5c972c9c841f98dc3fad3455d3084dba61
+    0.18.1: sha256:e6556fc7ccd071d7612446945d361c869dfeb423e0738147e0b46b2550bc2bf9
+    0.18.0: sha256:fd20a79875c8089694fb9b5dc3a0bf89d51711f9239637931ff0ace76ce78816
+    0.17.0: sha256:dee29ad27f3958882b450019e2021698282e8fcf8b136c27397798102cc1ad13
     0.16.24: sha256:cf7f1276bbcf4aa5e6347d5619efe990cf1340d5898f8405931e277a1f76c670
     0.16.23: sha256:7973302bead01c3f2e1d0f03e2766a0d6e76d3c52c666c750b9871a28b9afb32
     0.16.22: sha256:b70c15e40b36ac34d59597f2448c5b4e0033964c517f926dbb9654aa07fb1e5b
@@ -591,6 +596,11 @@ ciliumcli_binary_checksums:
     0.15.16: sha256:86ed6a2e796c39dd00072e7c141fc35b68d63392d1ac5e183a7ce9d7263e23a0
     0.15.15: sha256:5c1693ea163b094a92ebc6997b6e678cc8c24a52040c22433b58b419de74b28f
   amd64:
+    0.18.3: sha256:5fe565f3b98b5846b867319aa76bc057fca37894d80db56edc20e4e809d10b25
+    0.18.2: sha256:1b4bd5fd5c96ab1195cd4eb56841c983a21149c62ee39922b7955f1cd0eda23a
+    0.18.1: sha256:c472639d460173e8d807a3f57048f9d1bcdb325e9edba320550d7ec62b72f956
+    0.18.0: sha256:3ac8bd270763e40a7853c73f8c7ec9e49707e1723801884a083dc25469b6b4ba
+    0.17.0: sha256:4ba0687ff7d47e182a7328409fb0eae123e64fa6099cd6f8b9bf240c0012ecf4
     0.16.24: sha256:019c9c765222b3db5786f7b3a0bff2cd62944a8ce32681acfb47808330f405a7
     0.16.23: sha256:e7cd3b982eca9b6214226536a147490ebb6ea3caad40d5a724daeea0bec5e3be
     0.16.22: sha256:8bd9faae272aef2e75c686a55de782018013098b66439a1ee0c8ff1e05c5d32c

roles/network_plugin/cilium/defaults/main.yml

@@ -1,17 +1,17 @@
 ---
-cilium_min_version_required: "1.10"
+cilium_min_version_required: "1.15"
 # Log-level
 cilium_debug: false
 
-cilium_mtu: ""
+cilium_mtu: "0"
 cilium_enable_ipv4: "{{ ipv4_stack }}"
 cilium_enable_ipv6: "{{ ipv6_stack }}"
 
 # Enable l2 announcement from cilium to replace Metallb Ref: https://docs.cilium.io/en/v1.14/network/l2-announcements/
 cilium_l2announcements: false
 
 # Cilium agent health port
-cilium_agent_health_port: "{%- if cilium_version is version('1.11.6', '>=') -%}9879{%- else -%}9876{%- endif -%}"
+cilium_agent_health_port: "9879"
 
 # Identity allocation mode selects how identities are shared between cilium
 # nodes by setting how they are stored. The options are "crd" or "kvstore".
@@ -26,7 +26,7 @@ cilium_agent_health_port: "{%- if cilium_version is version('1.11.6', '>=') -%}9
 #   - --synchronize-k8s-nodes
 #   - --identity-allocation-mode=kvstore
 #   - Ref: https://docs.cilium.io/en/stable/internals/cilium_operator/#kvstore-operations
-cilium_identity_allocation_mode: kvstore
+cilium_identity_allocation_mode: crd
 
 # Etcd SSL dirs
 cilium_cert_dir: /etc/cilium/certs
@@ -55,20 +55,20 @@ cilium_enable_prometheus: false
 cilium_enable_portmap: false
 # Monitor aggregation level (none/low/medium/maximum)
 cilium_monitor_aggregation: medium
-# Kube Proxy Replacement mode (strict/partial)
-cilium_kube_proxy_replacement: partial
+# Kube Proxy Replacement mode (true/false)
+cilium_kube_proxy_replacement: false
+
+# If not defined `cilium_dns_proxy_enable_transparent_mode`, it will following the Cilium behavior.
+# When Cilium is configured to replace kube-proxy, it automatically enables dnsProxy, which will conflict with nodelocaldns.
+# You can set `false` avoid conflict with nodelocaldns.
+# https://github.com/cilium/cilium/issues/33144
+# cilium_dns_proxy_enable_transparent_mode:
 
 # If upgrading from Cilium < 1.5, you may want to override some of these options
 # to prevent service disruptions. See also:
 # http://docs.cilium.io/en/stable/install/upgrade/#changes-that-may-require-action
 cilium_preallocate_bpf_maps: false
 
-# `cilium_tofqdns_enable_poller` is deprecated in 1.8, removed in 1.9
-cilium_tofqdns_enable_poller: false
-
-# `cilium_enable_legacy_services` is deprecated in 1.6, removed in 1.9
-cilium_enable_legacy_services: false
-
 # Auto direct nodes routes can be used to advertise pods routes in your cluster
 # without any tunelling (with `cilium_tunnel_mode` sets to `disabled`).
 # This works only if you have a L2 connectivity between all your nodes.
@@ -100,8 +100,8 @@ cilium_encryption_enabled: false
 cilium_encryption_type: "ipsec"
 
 # Enable encryption for pure node to node traffic.
-# This option is only effective when `cilium_encryption_type` is set to `ipsec`.
-cilium_ipsec_node_encryption: false
+# This option is only effective when `cilium_encryption_type` is set to `wireguard`.
+cilium_encryption_node_encryption: false
 
 # If your kernel or distribution does not support WireGuard, Cilium agent can be configured to fall back on the user-space implementation.
 # When this flag is enabled and Cilium detects that the kernel has no native support for WireGuard,
@@ -115,6 +115,7 @@ cilium_wireguard_userspace_fallback: false
 # In case they select the Pod at egress, then the bandwidth enforcement will be disabled for those Pods.
 # Bandwidth Manager requires a v5.1.x or more recent Linux kernel.
 cilium_enable_bandwidth_manager: false
+cilium_enable_bandwidth_manager_bbr: false
 
 # IP Masquerade Agent
 # https://docs.cilium.io/en/stable/concepts/networking/masquerading/
@@ -137,6 +138,7 @@ cilium_non_masquerade_cidrs:
 ### Indicates whether to masquerade traffic to the link local prefix.
 ### If the masqLinkLocal is not set or set to false, then 169.254.0.0/16 is appended to the non-masquerade CIDRs list.
 cilium_masq_link_local: false
+cilium_masq_link_local_ipv6: false
 ### A time interval at which the agent attempts to reload config from disk
 cilium_ip_masq_resync_interval: 60s
 
@@ -145,10 +147,10 @@ cilium_ip_masq_resync_interval: 60s
 cilium_enable_hubble: false
 ### Enable Hubble-ui
 cilium_enable_hubble_ui: "{{ cilium_enable_hubble }}"
-### Enable Hubble Metrics
+### Enable Hubble Metrics (deprecated)
 cilium_enable_hubble_metrics: false
 ### if cilium_enable_hubble_metrics: true
-cilium_hubble_metrics: {}
+cilium_hubble_metrics: []
 # - dns
 # - drop
 # - tcp
@@ -160,12 +162,25 @@ cilium_hubble_install: false
 ### Enable auto generate certs if cilium_hubble_install: true
 cilium_hubble_tls_generate: false
 
+cilium_hubble_export_file_max_backups: "5"
+cilium_hubble_export_file_max_size_mb: "10"
+
+cilium_hubble_export_dynamic_enabled: false
+cilium_hubble_export_dynamic_config_content:
+  - name: all
+    fieldMask: []
+    includeFilters: []
+    excludeFilters: []
+    filePath: "/var/run/cilium/hubble/events.log"
+
 ### Capacity of Hubble events buffer. The provided value must be one less than an integer power of two and no larger than 65535
 ### (ie: 1, 3, ..., 2047, 4095, ..., 65535) (default 4095)
 # cilium_hubble_event_buffer_capacity: 4095
 ### Buffer size of the channel to receive monitor events.
 # cilium_hubble_event_queue_size: 50
 
+cilium_gateway_api_enabled: false
+
 # The default IP address management mode is "Cluster Scope".
 # https://docs.cilium.io/en/stable/concepts/networking/ipam/
 cilium_ipam_mode: cluster-pool
@@ -190,7 +205,8 @@ cilium_ipam_mode: cluster-pool
 
 
 # Extra arguments for the Cilium agent
-cilium_agent_custom_args: []
+cilium_agent_custom_args: [] # deprecated
+cilium_agent_extra_args: []
 
 # For adding and mounting extra volumes to the cilium agent
 cilium_agent_extra_volumes: []
@@ -214,13 +230,19 @@ cilium_operator_extra_volumes: []
 cilium_operator_extra_volume_mounts: []
 
 # Extra arguments for the Cilium Operator
-cilium_operator_custom_args: []
+cilium_operator_custom_args: [] # deprecated
+cilium_operator_extra_args: []
 
 # Tolerations of the cilium operator
 cilium_operator_tolerations:
   - operator: "Exists"
 
+# Unique ID of the cluster. Must be unique across all connected
+# clusters and in the range of 1 to 255. Only required for Cluster Mesh,
+# may be 0 if Cluster Mesh is not used.
+cilium_cluster_id: 0
 # Name of the cluster. Only relevant when building a mesh of clusters.
+# The "default" name cannot be used if the Cluster ID is different from 0.
 cilium_cluster_name: default
 
 # Make Cilium take ownership over the `/etc/cni/net.d` directory on the node, renaming all non-Cilium CNI configurations to `*.cilium_bak`.
@@ -263,7 +285,7 @@ cilium_enable_bpf_masquerade: false
 # host stack (true) or directly and more efficiently out of BPF (false) if
 # the kernel supports it. The latter has the implication that it will also
 # bypass netfilter in the host namespace.
-cilium_enable_host_legacy_routing: true
+cilium_enable_host_legacy_routing: false
 
 # -- Enable use of the remote node identity.
 # ref: https://docs.cilium.io/en/v1.7/install/upgrade/#configmap-remote-node-identity
@@ -307,9 +329,9 @@ cilium_rolling_restart_wait_retries_count: 30
 cilium_rolling_restart_wait_retries_delay_seconds: 10
 
 # Cilium changed the default metrics exporter ports in 1.12
-cilium_agent_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9962', '9090') }}"
-cilium_operator_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9963', '6942') }}"
-cilium_hubble_scrape_port: "{{ cilium_version is version('1.12', '>=') | ternary('9965', '9091') }}"
+cilium_agent_scrape_port: "9962"
+cilium_operator_scrape_port: "9963"
+cilium_hubble_scrape_port: "9965"
 
 # Cilium certgen args for generate certificate for hubble mTLS
 cilium_certgen_args:
@@ -328,26 +350,5 @@ cilium_certgen_args:
   hubble-relay-client-cert-secret-name: hubble-relay-client-certs
   hubble-relay-server-cert-generate: false
 
-# A list of extra rules variables to add to clusterrole for cilium operator, formatted like:
-#   cilium_clusterrole_rules_operator_extra_vars:
-#     - apiGroups:
-#       - '""'
-#       resources:
-#       - pods
-#       verbs:
-#       - delete
-#     - apiGroups:
-#       - '""'
-#       resources:
-#       - nodes
-#       verbs:
-#       - list
-#       - watch
-#       resourceNames:
-#       - toto
-cilium_clusterrole_rules_operator_extra_vars: []
 cilium_enable_host_firewall: false
 cilium_policy_audit_mode: false
-
-cilium_hubble_export_file_max_backups: "5"
-cilium_hubble_export_file_max_size_mb: "10"

roles/network_plugin/cilium/tasks/apply.yml

@@ -1,14 +1,7 @@
 ---
-- name: Cilium | Start Resources
-  kube:
-    name: "{{ item.item.name }}"
-    namespace: "kube-system"
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "{{ item.item.type }}"
-    filename: "{{ kube_config_dir }}/{{ item.item.name }}-{{ item.item.file }}"
-    state: "latest"
-  loop: "{{ cilium_node_manifests.results }}"
-  when: inventory_hostname == groups['kube_control_plane'][0] and not item is skipped
+- name: Cilium | Install
+  command: "{{ bin_dir }}/cilium install --version {{ cilium_version }} -f {{ kube_config_dir }}/cilium-values.yaml"
+  when: inventory_hostname == groups['kube_control_plane'][0]
 
 - name: Cilium | Wait for pods to run
   command: "{{ kubectl }} -n kube-system get pods -l k8s-app=cilium -o jsonpath='{.items[?(@.status.containerStatuses[0].ready==false)].metadata.name}'"  # noqa literal-compare
@@ -19,19 +12,6 @@
   failed_when: false
   when: inventory_hostname == groups['kube_control_plane'][0]
 
-- name: Cilium | Hubble install
-  kube:
-    name: "{{ item.item.name }}"
-    namespace: "kube-system"
-    kubectl: "{{ bin_dir }}/kubectl"
-    resource: "{{ item.item.type }}"
-    filename: "{{ kube_config_dir }}/addons/hubble/{{ item.item.name }}-{{ item.item.file }}"
-    state: "latest"
-  loop: "{{ cilium_hubble_manifests.results }}"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0] and not item is skipped
-    - cilium_enable_hubble and cilium_hubble_install
-
 - name: Cilium | Wait for CiliumLoadBalancerIPPool CRD to be present
   command: "{{ kubectl }} wait --for condition=established --timeout=60s crd/ciliumloadbalancerippools.cilium.io"
   register: cillium_lbippool_crd_ready

roles/network_plugin/cilium/tasks/check.yml

@@ -48,7 +48,7 @@
     msg: "cilium_encryption_type must be either 'ipsec' or 'wireguard'"
   when: cilium_encryption_enabled
 
-- name: Stop if cilium_version is < 1.10.0
+- name: Stop if cilium_version is < {{ cilium_min_version_required }}
   assert:
     that: cilium_version is version(cilium_min_version_required, '>=')
     msg: "cilium_version is too low. Minimum version {{ cilium_min_version_required }}"

roles/network_plugin/cilium/tasks/install.yml

@@ -30,64 +30,20 @@
   when:
     - cilium_identity_allocation_mode == "kvstore"
 
-- name: Cilium | Create hubble dir
-  file:
-    path: "{{ kube_config_dir }}/addons/hubble"
-    state: directory
-    owner: root
-    group: root
-    mode: "0755"
-  when:
-    - inventory_hostname == groups['kube_control_plane'][0]
-    - cilium_hubble_install
-
-- name: Cilium | Create Cilium node manifests
+- name: Cilium | Enable portmap addon
   template:
-    src: "{{ item.name }}/{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/{{ item.name }}-{{ item.file }}"
+    src: 000-cilium-portmap.conflist.j2
+    dest: /etc/cni/net.d/000-cilium-portmap.conflist
     mode: "0644"
-  loop:
-    - {name: cilium, file: config.yml, type: cm}
-    - {name: cilium-operator, file: crb.yml, type: clusterrolebinding}
-    - {name: cilium-operator, file: cr.yml, type: clusterrole}
-    - {name: cilium, file: crb.yml, type: clusterrolebinding}
-    - {name: cilium, file: cr.yml, type: clusterrole}
-    - {name: cilium, file: secret.yml, type: secret, when: "{{ cilium_encryption_enabled and cilium_encryption_type == 'ipsec' }}"}
-    - {name: cilium, file: ds.yml, type: ds}
-    - {name: cilium-operator, file: deploy.yml, type: deploy}
-    - {name: cilium-operator, file: sa.yml, type: sa}
-    - {name: cilium, file: sa.yml, type: sa}
-  register: cilium_node_manifests
-  when:
-    - ('kube_control_plane' in group_names)
-    - item.when | default(True) | bool
+  when: cilium_enable_portmap
 
-- name: Cilium | Create Cilium Hubble manifests
+- name: Cilium | Render values
   template:
-    src: "{{ item.name }}/{{ item.file }}.j2"
-    dest: "{{ kube_config_dir }}/addons/hubble/{{ item.name }}-{{ item.file }}"
+    src: values.yaml.j2
+    dest: "{{ kube_config_dir }}/cilium-values.yaml"
     mode: "0644"
-  loop:
-    - {name: hubble, file: config.yml, type: cm}
-    - {name: hubble, file: crb.yml, type: clusterrolebinding}
-    - {name: hubble, file: cr.yml, type: clusterrole}
-    - {name: hubble, file: cronjob.yml, type: cronjob, when: "{{ cilium_hubble_tls_generate }}"}
-    - {name: hubble, file: deploy.yml, type: deploy}
-    - {name: hubble, file: job.yml, type: job, when: "{{ cilium_hubble_tls_generate }}"}
-    - {name: hubble, file: sa.yml, type: sa}
-    - {name: hubble, file: service.yml, type: service}
-  register: cilium_hubble_manifests
   when:
     - inventory_hostname == groups['kube_control_plane'][0]
-    - cilium_enable_hubble and cilium_hubble_install
-    - item.when | default(True) | bool
-
-- name: Cilium | Enable portmap addon
-  template:
-    src: 000-cilium-portmap.conflist.j2
-    dest: /etc/cni/net.d/000-cilium-portmap.conflist
-    mode: "0644"
-  when: cilium_enable_portmap
 
 - name: Cilium | Copy Ciliumcli binary from download dir
   copy: