Add flavor to run Gardener in the mini-lab (#202)

Author: Gerrit91

.github/workflows/integration.yaml

@@ -21,6 +21,7 @@ jobs:
         flavors:
           - name: cumulus
           - name: sonic
+          - name: gardener
 
     steps:
     - name: Gain back workspace permissions # https://github.com/actions/checkout/issues/211

.gitignore

@@ -12,3 +12,7 @@ requirements.yaml
 .extra_vars.yaml
 sonic-vs.img
 *.bak
+.ansible
+files/certs/*.pem
+files/certs/**/*.pem
+files/certs/**/*.crt

Makefile

@@ -34,6 +34,13 @@ else ifeq ($(MINI_LAB_FLAVOR),capms)
 LAB_MACHINES=machine01,machine02,machine03
 LAB_TOPOLOGY=mini-lab.capms.yaml
 VRF=Vrf20
+else ifeq ($(MINI_LAB_FLAVOR),gardener)
+GARDENER_ENABLED=true
+# usually gardener restricts the maximum version for k8s:
+K8S_VERSION=1.30.8
+LAB_MACHINES=machine01,machine02
+LAB_TOPOLOGY=mini-lab.sonic.yaml
+VRF=Vrf20
 else
 $(error Unknown flavor $(MINI_LAB_FLAVOR))
 endif
@@ -50,7 +57,7 @@ else
 endif
 
 .PHONY: up
-up: env control-plane-bake partition-bake
+up: env gen-certs control-plane-bake partition-bake
 	@chmod 600 files/ssh/id_rsa
 	docker compose up --remove-orphans --force-recreate control-plane partition
 	@$(MAKE)	--no-print-directory	start-machines
@@ -67,6 +74,21 @@ restart: down up
 .PHONY: down
 down: cleanup
 
+.PHONY: gen-certs
+gen-certs:
+	@if ! [ -f "files/certs/ca.pem" ]; then \
+		echo "certificate generation required, running cfssl container"; \
+		docker run --rm \
+			--user $$(id -u):$$(id -g) \
+			--entrypoint bash \
+			-v ${PWD}:/work \
+			cfssl/cfssl /work/scripts/roll_certs.sh; fi
+
+.PHONY: roll-certs
+roll-certs:
+	rm files/certs/ca.pem
+	$(MAKE) gen-certs
+
 .PHONY: control-plane
 control-plane: control-plane-bake env
 	docker compose up --remove-orphans --force-recreate control-plane
@@ -272,3 +294,20 @@ dev-env:
 	@echo "export METALCTL_API_URL=http://api.172.17.0.1.nip.io:8080/metal"
 	@echo "export METALCTL_HMAC=metal-admin"
 	@echo "export KUBECONFIG=$(KUBECONFIG)"
+
+## Gardener integration
+
+.PHONY: fetch-virtual-kubeconfig
+fetch-virtual-kubeconfig:
+	kubectl config unset users.virtual-garden
+	kubectl config unset contexts.virtual-garden
+	kubectl config unset clusters.virtual-garden
+	kubectl get secret -n garden garden-kubeconfig-for-admin -o jsonpath='{.data.kubeconfig}' | base64 -d > .virtual-kubeconfig
+	kubectl --kubeconfig=.virtual-kubeconfig config rename-context garden virtual-garden
+	sed -i 's/name: garden/name: virtual-garden/g' .virtual-kubeconfig
+	sed -i 's/name: admin/name: virtual-garden/g' .virtual-kubeconfig
+	kubectl --kubeconfig=.virtual-kubeconfig config set contexts.virtual-garden.cluster virtual-garden
+	kubectl --kubeconfig=.virtual-kubeconfig config set contexts.virtual-garden.user virtual-garden
+	KUBECONFIG=$$KUBECONFIG:.virtual-kubeconfig kubectl config view --flatten > .merged-kubeconfig
+	rm .virtual-kubeconfig
+	mv .merged-kubeconfig .kubeconfig

README.md

@@ -196,8 +196,10 @@ docker compose run --rm metalctl machine rm 00000000-0000-0000-0000-000000000001
 
 There are two versions, or flavors, of the mini-lab environment which differ in regards to the NOS running on the leaves:
 
-- `cumulus` -- runs 2 Cumulus switches.
-- `sonic` -- runs 2 SONiC switches
+- `cumulus`: runs 2 Cumulus switches.
+- `sonic`: runs 2 SONiC switches
+- `capms`: runs the SONiC flavor but with three instead of two machines (this is used for  [cluster-provider-metal-stack](https://github.com/metal-stack/cluster-api-provider-metal-stack) in order to have dedicated hosts for control plane / worker / firewall)
+- `gardener`: installs the [Gardener](https://gardener.cloud) in the mini-lab
 
 In order to start specific flavor, you can define the flavor as follows:
 

compose.yaml

@@ -19,12 +19,9 @@ services:
       - CI=${CI}
       - DOCKER_HUB_USER=${DOCKER_HUB_USER}
       - DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN}
+      - GARDENER_ENABLED=${GARDENER_ENABLED:-}
     network_mode: host
     working_dir: /mini-lab
-    dns:
-      - 172.17.0.1
-      - 1.1.1.1
-      - 1.0.0.1
     entrypoint:
       - /bin/bash
       - -ce
@@ -55,10 +52,6 @@ services:
       - DOCKER_HUB_TOKEN=${DOCKER_HUB_TOKEN}
     network_mode: host
     working_dir: /mini-lab
-    dns:
-      - 172.17.0.1
-      - 1.1.1.1
-      - 1.0.0.1
     entrypoint:
       - /bin/bash
       - -ce
@@ -82,8 +75,4 @@ services:
       - ./files/ignition.json:/tmp/ignition.json
       - ./files/rules.yaml:/tmp/rules.yaml
     network_mode: host
-    dns:
-      - 172.17.0.1
-      - 1.1.1.1
-      - 1.0.0.1
     command: --version

deploy_control_plane.yaml

@@ -3,10 +3,6 @@
   hosts: control-plane
   connection: local
   gather_facts: false
-  vars:
-    setup_yaml:
-      - url: https://raw.githubusercontent.com/metal-stack/releases/{{ metal_stack_release_version }}/release.yaml
-        meta_var: metal_stack_release
   roles:
     - name: ansible-common
       tags: always
@@ -26,3 +22,7 @@
       tags: auditing
     - name: metal-roles/control-plane/roles/metal
       tags: metal
+
+- name: deploy gardener
+  import_playbook: deploy_gardener.yaml
+  when: gardener_enabled

deploy_gardener.yaml

@@ -0,0 +1,82 @@
+---
+- name: deploy gardener
+  hosts: control-plane
+  connection: local
+  gather_facts: false
+  pre_tasks:
+    - name: Fake Gardener metal shoot
+      k8s:
+        definition:
+          apiVersion: v1
+          kind: ConfigMap
+          metadata:
+            name: shoot-info
+            namespace: kube-system
+          data:
+            nodeNetwork: 172.18.0.0/16
+            podNetwork: 10.244.0.0/24
+            serviceNetwork: 10.96.0.0/16
+
+    - name: Create garden namespace
+      k8s:
+        definition:
+          apiVersion: v1
+          kind: Namespace
+          metadata:
+            name: garden
+
+    # our current state in metal-roles/gardener does not support network policies from gardenlet <-> virtual garden
+    # this should be possible to resolve when we use the Gardener Operator
+    - name: Deploy allow all network policy
+      k8s:
+        definition: "{{ lookup('file', 'netpol-allow-all.yaml') }}"
+        namespace: garden
+        apply: yes
+  roles:
+    - name: ansible-common
+      tags: always
+    - name: minio
+    - name: powerdns
+      tags: powerdns
+    - name: metal-roles/control-plane/roles/gardener
+      tags: gardener
+  vars:
+    metal_control_plane_host_provider: metal
+
+  post_tasks:
+    # gardener exposes the istio ingress gateway through service type load balancer
+    # we can fake the exposal by patching the status field, which is also what's
+    # done in the gardener local environment
+    - name: Wait for istio ingress gateway service
+      kubernetes.core.k8s_info:
+        api_version: v1
+        kind: Service
+        name: istio-ingressgateway
+        namespace: istio-ingress
+      register: result
+      until: result.resources
+      retries: 30
+      delay: 10
+
+    - name: Patch ingress status of istio ingress gateway to allow seed to get ready
+      patch_service_status_k8s:
+        name: istio-ingressgateway
+        namespace: istio-ingress
+        body:
+          status:
+            loadBalancer:
+              ingress:
+                - ip: "172.17.0.1"
+
+    - name: Wait until seed is ready
+      kubernetes.core.k8s_info:
+        api_version: "core.gardener.cloud/v1beta1"
+        kind: Seed
+        name: "{{ metal_control_plane_stage_name }}"
+        kubeconfig: "{{ gardener_kube_apiserver_kubeconfig_path }}"
+        wait: yes
+        wait_condition:
+          reason: GardenletReady
+          status: "True"
+          type: GardenletReady
+        wait_timeout: 300

deploy_partition.yaml

@@ -29,6 +29,22 @@
       command: tar xf /root/jessie-apt-transport-fix.tar.gz
     - name: install apt-transport
       shell: dpkg -i /root/https-deps/*.deb && apt-get install -fy
+    - name: use own repo mirrors (old ones are 404)
+      copy:
+        dest: /etc/apt/sources.list
+        content: |
+          deb https://cumulus-3.apt.metal-stack.io/cumulus CumulusLinux-3 main
+          deb-src https://cumulus-3.apt.metal-stack.io/cumulus CumulusLinux-3 main
+
+          deb https://cumulus-3.apt.metal-stack.io/cumulus-updates CumulusLinux-3-updates main
+          deb-src https://cumulus-3.apt.metal-stack.io/cumulus-updates CumulusLinux-3-updates main
+
+          deb https://cumulus-3.apt.metal-stack.io/cumulus-security-updates CumulusLinux-3-security-updates main
+          deb-src https://cumulus-3.apt.metal-stack.io/cumulus-security-updates CumulusLinux-3-security-updates main
+    - name: set resolv.conf
+      shell: rm /etc/resolv.conf && echo 'nameserver 8.8.8.8' > /etc/resolv.conf
+    - name: use own repo key
+      shell: curl -fsSL https://cumulus-3.apt.metal-stack.io/cumulus-3-repo.gpg | apt-key add -
   roles:
     - name: cumulus
       tags: cumulus
@@ -37,10 +53,6 @@
 
 - name: Deploy dhcp server and pixiecore on leaf01
   hosts: leaf01
-  vars:
-    setup_yaml:
-      - url: https://raw.githubusercontent.com/metal-stack/releases/{{ metal_stack_release_version }}/release.yaml
-        meta_var: metal_stack_release
   roles:
     - name: ansible-common
       tags: always
@@ -51,10 +63,6 @@
 
 - name: Deploy metal-core
   hosts: leaves
-  vars:
-    setup_yaml:
-      - url: https://raw.githubusercontent.com/metal-stack/releases/{{ metal_stack_release_version }}/release.yaml
-        meta_var: metal_stack_release
   roles:
     - name: ansible-common
       tags: always
@@ -65,10 +73,6 @@
   hosts: localhost
   connection: local
   gather_facts: no
-  vars:
-    setup_yaml:
-      - url: https://raw.githubusercontent.com/metal-stack/releases/{{ metal_stack_release_version }}/release.yaml
-        meta_var: metal_stack_release
   roles:
     - name: ansible-common
       tags: always

files/certs/ca-csr.json

@@ -9,7 +9,7 @@
     {
       "C":  "DE",
       "L":  "Munich",
-      "O":  "Metal-Stack",
+      "O":  "metal-stack",
       "OU": "DevOps",
       "ST": "Bavaria"
     }

files/certs/ca-key.pem

@@ -0,0 +1,17 @@
+{
+  "CN": "client",
+  "hosts": [""],
+  "key": {
+    "algo": "ecdsa",
+    "size": 256
+  },
+  "names": [
+    {
+      "C":  "DE",
+      "L":  "Munich",
+      "O":  "metal-stack",
+      "OU": "DevOps",
+      "ST": "Bavaria"
+    }
+  ]
+}