Merge pull request #10 from garutilorenzo/bugfix/kubernetes_repo

Repo refactor for new k8s version

Author: garutilorenzo

README.md

@@ -7,7 +7,17 @@
 
 This ansible role will install and configure a high available Kubernetes cluster. This repo automate the installation process of Kubernetes using [kubeadm](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/).
 
-This repo is only a example on how to use Ansible automation to install and configure a Kubernetes cluster. For a production environment use [Kubespray](https://kubernetes.io/docs/setup/production-environment/tools/kubespray/)
+This repo is only a example on how to use Ansible automation to install and configure a Kubernetes cluster. For a production environment use [Kubespray](https://kubespray.io)
+
+## Supported OS
+
+This role was tested with:
+
+Ubuntu: 22.04, 24.04
+
+Fedora: 40
+
+Tested partially  with RHEL 8.
 
 ## Requirements
 
@@ -27,31 +37,35 @@ ansible-galaxy install git+https://github.com/garutilorenzo/ansible-role-linux-k
 
 This role accept this variables:
 
-| Var   | Required |  Default | Desc |
-| ------- | ------- | ----------- |  ----------- |
-| `kubernetes_subnet`       | `yes`       |  `192.168.25.0/24` | Subnet where Kubernetess will be deployed. If the VM or bare metal server has more than one interface, Ansible will filter the interface used by Kubernetes based on the interface subnet |
-| `disable_firewall`       | `no`       | `no`       | If set to yes Ansible will disable the firewall.   |
-| `kubernetes_version`       | `no`       | `1.25.0`       | Kubernetes version to install  |
-| `kubernetes_cri`       | `no`       | `containerd`       | Kubernetes [CRI](https://kubernetes.io/docs/concepts/architecture/cri/) to install.   |
-| `kubernetes_cni`       | `no`       | `flannel`       | Kubernetes [CNI](https://github.com/containernetworking/cni) to install.  |
-| `kubernetes_dns_domain`       | `no`       | `cluster.local`       | Kubernetes default DNS domain  |
-| `kubernetes_pod_subnet`       | `no`       | `10.244.0.0/16`       | Kubernetes pod subnet  |
-| `kubernetes_service_subnet`       | `no`       | `10.96.0.0/12`       | Kubernetes service subnet  |
-| `kubernetes_api_port`       | `no`       | `6443`       | kubeapi listen port  |
-| `setup_vip`       | `no`       | `no`       | Setup kubernetes VIP addres using [kube-vip](https://kube-vip.io/)   |
-| `kubernetes_vip_ip`       | `no`       | `192.168.25.225`       | **Required** if setup_vip is set to *yes*. Vip ip address for the control plane  |
-| `kubevip_version`       | `no`       | `v0.4.3`       | kube-vip container version  |
-| `install_longhorn`       | `no`       | `no`       | Install [Longhorn](#longhorn), Cloud native distributed block storage for Kubernetes.  |
-| `longhorn_version`       | `no`       | `v1.3.1`       | Longhorn release.  |
-| `install_nginx_ingress`       | `no`       | `no`       | Install [nginx ingress controller](#nginx-ingress-controller)  |
-| `nginx_ingress_controller_version`       | `no`       | `controller-v1.3.0`       | nginx ingress controller version  |
-| `nginx_ingress_controller_http_nodeport`       | `no`       | `30080`       | NodePort used by nginx ingress controller for the incoming http traffic  |
-| `nginx_ingress_controller_https_nodeport`       | `no`       | `30443`       |  NodePort used by nginx ingress controller for the incoming https traffic  |
-| `enable_nginx_ingress_proxy_protocol`       | `no`       | `no`       | Enable  nginx ingress controller proxy protocol mode |
-| `enable_nginx_real_ip`       | `no`       | `no`       | Enable nginx ingress controller real-ip module |
-| `nginx_ingress_real_ip_cidr`       | `no`       | `0.0.0.0/0`       | **Required** if enable_nginx_real_ip is set to *yes* Trusted subnet to use with the real-ip module  |
-| `nginx_ingress_proxy_body_size`       | `no`       | `20m`       | nginx ingress controller max proxy body size  |
-| `sans_base`       | `no`       | `[list of values, see defaults/main.yml]`       | list of ip addresses or FQDN uset to sign the kube-api certificate  |
+| Var  |  Default | Desc |
+| ------- | ----------- |  ----------- |
+| `disable_firewall`       | If set to yes Ansible will disable the firewall.   |
+| `disable_selinux`        | `yes`       | If set to yes Ansible will disable Selinux on RedHat based distro. Default `yes` [Ref.](https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/)  |
+| `kubernetes_version`       | `1.31.4`       | Kubernetes version to install  |
+| `kubernetes_image_registry`       | `registry.k8s.io`       | Default k8s registry  |
+| `kubernetes_pause_image`       | `registry.k8s.io/pause:3.10`       | Default `pause` image version (includes registry).  Needed to solve [this](https://github.com/kubernetes/kubeadm/issues/3146) issue. |
+| `kubernetes_cri`       | `containerd`       | Kubernetes [CRI](https://kubernetes.io/docs/concepts/architecture/cri/) to install.   |
+| `kubernetes_cni`       | `flannel`       | Kubernetes [CNI](https://github.com/containernetworking/cni) to install.  |
+| `flannel_version`       | `v0.26.2`       | Required if  `kubernetes_cni` is set to `flannel`. Documentation available [here](https://github.com/flannel-io/flannel) |
+| `kubernetes_dns_domain`       | `cluster.local`       | Kubernetes default DNS domain  |
+| `kubernetes_pod_subnet`       | `10.244.0.0/16`       | Kubernetes pod subnet  |
+| `kubernetes_service_subnet`       | `10.96.0.0/12`       | Kubernetes service subnet  |
+| `kubernetes_api_port`       | `6443`       | kubeapi listen port  |
+| `kubernetes_subnet`       |  `192.168.25.0/24` | Subnet where Kubernetess will be deployed. If the VM or bare metal server has more than one interface, Ansible will filter the interface used by Kubernetes based on the interface subnet |
+| `setup_vip`       | Setup kubernetes VIP addres using [kube-vip](https://kube-vip.io/)   |
+| `kubernetes_vip_ip`       | `192.168.25.225`       | **Required** if setup_vip is set to *yes*. Vip ip address for the control plane  |
+| `kubevip_version`       | `v0.8.7`       | kube-vip container version  |
+| `install_longhorn`       | Install [Longhorn](#longhorn), Cloud native distributed block storage for Kubernetes.  |
+| `longhorn_version`       | `v1.7.2`       | Longhorn release.  |
+| `install_nginx_ingress`       | Install [nginx ingress controller](#nginx-ingress-controller)  |
+| `nginx_ingress_controller_version`       | `v1.12.0`       | nginx ingress controller version  |
+| `nginx_ingress_controller_http_nodeport`       | `30080`       | NodePort used by nginx ingress controller for the incoming http traffic  |
+| `nginx_ingress_controller_https_nodeport`       | `30443`       |  NodePort used by nginx ingress controller for the incoming https traffic  |
+| `enable_nginx_ingress_proxy_protocol`       | Enable  nginx ingress controller proxy protocol mode |
+| `enable_nginx_real_ip`       | Enable nginx ingress controller real-ip module |
+| `nginx_ingress_real_ip_cidr`       | `0.0.0.0/0`       | **Required** if enable_nginx_real_ip is set to *yes* Trusted subnet to use with the real-ip module  |
+| `nginx_ingress_proxy_body_size`       | `20m`       | nginx ingress controller max proxy body size  |
+| `sans_base`       | `[list of values, see defaults/main.yml]`       | list of ip addresses or FQDN uset to sign the kube-api certificate  |
 
 ## Extra Variables
 
@@ -86,7 +100,10 @@ In the Vagrantfile you can inject your public ssh key directly in the authorized
 
 ## Using this role
 
-To use this role you follow the example in the [examples/](examples/) dir. Adjust the hosts.ini file with your hosts and run the playbook:
+To use this role you follow the example in the [examples/](examples/) dir. 
+On the very first run add the extra variable parameter `-e kubernetes_init_host=<HOSTNAME>` to the `ansible-playbook` command where HOSTNAME is the hostname where the k8s cluster will be initialized.
+
+Adjust the example hosts.ini file with your hosts and run the playbook:
 
 ```
 lorenzo@mint-virtual:~$ ansible-playbook -i hosts-ubuntu.ini site.yml -e kubernetes_init_host=k8s-ubuntu-0

defaults/main.yml

@@ -1,8 +1,11 @@
 ---
 
 disable_firewall: no
+disable_selinux: yes # Ref. https://kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/
 
-kubernetes_version: 1.27.3
+kubernetes_version: 1.31.4
+kubernetes_image_registry: registry.k8s.io
+kubernetes_pause_image: registry.k8s.io/pause:3.10
 kubernetes_cri: containerd
 kubernetes_cni: flannel
 kubernetes_dns_domain: cluster.local
@@ -15,13 +18,14 @@ kubernetes_subnet: 192.168.25.0/24
 setup_vip: no
 kubernetes_vip_ip: 192.168.25.225
 
-kubevip_version: v0.6.0
+kubevip_version: v0.8.7
+flannel_version: v0.26.2
 
 install_longhorn: no
-longhorn_version: v1.4.3
+longhorn_version: v1.7.2
 
 install_nginx_ingress: no
-nginx_ingress_controller_version: controller-v1.8.1
+nginx_ingress_controller_version: v1.12.0
 nginx_ingress_controller_http_nodeport: 30080
 nginx_ingress_controller_https_nodeport: 30443
 enable_nginx_ingress_proxy_protocol: no

examples/hosts.ini

@@ -10,4 +10,7 @@ k8s-ubuntu-2 ansible_host=192.168.25.112
 [kubeworker]
 k8s-ubuntu-3 ansible_host=192.168.25.113
 k8s-ubuntu-4 ansible_host=192.168.25.114
-k8s-ubuntu-5 ansible_host=192.168.25.115
+k8s-ubuntu-5 ansible_host=192.168.25.115
+
+[kubernetes:vars]
+ansible_python_interpreter=/usr/bin/python3

examples/site.yaml

@@ -1,4 +1,15 @@
 ---
+
+- hosts: kubernetes
+  become: yes
+  remote_user: vagrant
+  tasks:
+    - ansible.builtin.import_role:
+        name: ansible-role-linux-kubernetes
+        tasks_from: set_custom_fact
+  vars_files:
+    - vars.yaml
+
 - hosts: kubemaster
   become: yes
   remote_user: vagrant

examples/vars.yaml

@@ -1,8 +1,6 @@
 ---
 
 disable_firewall: yes
-kubernetes_subnet: 192.168.25.0/24 
 
-setup_vip: yes
 install_nginx_ingress: yes
 install_longhorn: yes

meta/main.yml

@@ -8,9 +8,13 @@ galaxy_info:
       versions:
         - jammy
         - focal
+        - noble
     - name: EL
       versions:
         - 8
+    - name: Fedora
+      versions:
+        - 40
   galaxy_tags:
     - kubernetes
     - k8s

tasks/init_cluster.yml

@@ -4,28 +4,10 @@
   ansible.builtin.shell: kubectl get nodes
   ignore_errors: true
   register: cluster_exist
-
-- set_fact:
-    kubernetes_init: "{% if kubernetes_init_host is defined and  kubernetes_init_host == inventory_hostname %}yes{% else %}no{% endif %}"
-
-- set_fact:
-    kubernetes_ip_address: "{{ item }}"
-  when: "item | ansible.utils.ipaddr( kubernetes_subnet )"
-  with_items: "{{ ansible_all_ipv4_addresses | difference([kubernetes_vip_ip]) }}"
-
-- set_fact:
-    kubernetes_image_repository: "registry.k8s.io"
 
 - block:
-
-    - set_fact:
-        kubernetes_iface: "{{ hostvars[inventory_hostname]['ansible_' + item ]['device'] }}"
-      when: 
-        - hostvars[inventory_hostname]['ansible_' + item ].ipv4 is defined 
-        - hostvars[inventory_hostname]['ansible_' + item ]['ipv4']['address'] | ansible.utils.ipaddr( kubernetes_subnet )
-      with_items: "{{ ansible_interfaces }}"
 
-    - set_fact:
+    - ansible.builtin.set_fact:
         apiserver_sans: "{{ (sans_base + [kubernetes_vip_ip] ) | unique }}"
 
     - name: render kubeadm-init.yml
@@ -39,7 +21,7 @@
         - setup_vip
         - inventory_hostname in groups['kubemaster']
 
-    - set_fact:
+    - ansible.builtin.set_fact:
         kubeadm_extra_args: "{% if groups['kubemaster'] | length > 1 %}--upload-certs{% else %}{% endif %}"
 
     - name: Init kubernetes cluster

tasks/install_cni.yml

@@ -3,14 +3,22 @@
 - block:
     - name: Download kube-flannel.yml
       ansible.builtin.get_url:
-        url: https://raw.githubusercontent.com/flannel-io/flannel/master/Documentation/kube-flannel.yml
+        url: https://raw.githubusercontent.com/flannel-io/flannel/refs/tags/{{ flannel_version }}/Documentation/kube-flannel.yml
         dest: /root/kube-flannel.yml
 
     - name: Set flannel iface
       ansible.builtin.lineinfile:
         path: /root/kube-flannel.yml
         insertafter: '^(\s*).*kube-subnet-mgr(.*)$'
         line: '        - --iface={{ kubernetes_iface }}'
+    
+    - name: Fix cni-plugin path only for Fedora distribution
+      ansible.builtin.replace:
+        path: /root/kube-flannel.yml
+        regexp: '^(\s*)path: \/opt\/cni\/bin(.*)$'
+        replace: '\1path: /usr/libexec/cni'
+      when: 
+        - ansible_distribution == 'Fedora'
 
     - name: Install flannel
       ansible.builtin.command: kubectl apply -f /root/kube-flannel.yml

tasks/install_cri.yml

@@ -23,8 +23,7 @@
 
     - name: Render containerd default config
       ansible.builtin.shell: containerd config default > /etc/containerd/config.toml
-      when: 
-        - containerd_config.found
+      when: containerd_config.found
 
     - name: enable SystemdCgroup for containerd
       ansible.builtin.replace:
@@ -34,6 +33,38 @@
       notify:
         - reload containerd
 
+    - name: Update the [grpc] block in /etc/containerd/config.toml
+      ansible.builtin.blockinfile:
+        path: /etc/containerd/config.toml
+        marker: "{mark}"
+        block: |2
+            address = "{{ cri_socket_paths[kubernetes_cri] }}"
+            gid = 0
+            max_recv_message_size = 16777216
+            max_send_message_size = 16777216
+            tcp_address = ""
+            tcp_tls_ca = ""
+            tcp_tls_cert = ""
+            tcp_tls_key = ""
+            uid = 0
+        marker_begin: "[grpc]"
+        marker_end: "[metrics]"
+      notify:
+        - reload containerd
+
+    # container runtime is inconsistent with that used by kubeadm
+    # Ref. https://github.com/kubernetes/kubeadm/issues/3146
+    - name: Update sandbox_image in /etc/containerd/config.toml
+      ansible.builtin.lineinfile:
+        path: /etc/containerd/config.toml
+        regexp: '^\s*sandbox_image\s*=.*'
+        line: '    sandbox_image = "{{ kubernetes_pause_image }}"'
+        
+    - name: render crictl.yaml
+      ansible.builtin.template:
+        src: crictl.yaml.j2
+        dest: /etc/crictl.yaml
+    
     - name: start and enable containerd
       ansible.builtin.systemd:
         name: containerd
@@ -42,49 +73,5 @@
 
   when: kubernetes_cri == 'containerd'
 
-# Fedora hack
-- name: Check if /opt/cni/bin directory exist
-  ansible.builtin.stat:
-    path: /opt/cni/bin
-  register: cni_bin_dir
-
-- block:
-    - name: Install containernetworking-plugins
-      ansible.builtin.package:
-        name: containernetworking-plugins
-        state: present
-
-    - name: stop containerd
-      ansible.builtin.systemd:
-        name: containerd
-        state: stopped
-
-    - name: Create /opt/cni/ directory
-      ansible.builtin.file: 
-        name: /opt/cni/
-        state: directory
-
-    - name: Remove /opt/cni/bin directory if exist
-      ansible.builtin.file: 
-        name: /opt/cni/bin
-        state: removed
-
-    - name: Create symlink from /usr/libexec/cni to /opt/cni/bin
-      ansible.builtin.file:
-        src: /usr/libexec/cni
-        dest: /opt/cni/bin
-        state: link
-
-    - name: start containerd
-      ansible.builtin.systemd:
-        name: containerd
-        state: started
-  when: 
-    - ansible_distribution == 'Fedora'
-    - kubernetes_cri == 'containerd'
-    - cni_bin_dir.stat.islnk is defined
-    - not cni_bin_dir.stat.islnk
-# END Fedora hack
-
 - name: Force all notified handlers to run now
   meta: flush_handlers

tasks/install_nginx_ingress.yml

@@ -9,7 +9,7 @@
 
 - block:
     - name: Install ingress-nginx
-      ansible.builtin.command: kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/{{ nginx_ingress_controller_version }}/deploy/static/provider/baremetal/deploy.yaml
+      ansible.builtin.command: kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-{{ nginx_ingress_controller_version }}/deploy/static/provider/baremetal/deploy.yaml
       delegate_to: "{{ groups['kubemaster'][0] }}"
       run_once: true