[ci] Add new example chart meant for testing and QA

.github/workflows/e2e-ci.yaml

@@ -0,0 +1,144 @@
+name: "E2E CI"
+
+on:
+  workflow_call:
+  workflow_dispatch:
+    inputs:
+      debug:
+        description: "Enable debug logs"
+        required: false
+        default: "false"
+      k3s_version:
+        description: "Version of k3s to use for the underlying cluster, should exist in https://hub.docker.com/r/rancher/k3s/tags"
+        required: false
+  pull_request:
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - '.gitignore'
+      - 'CODEOWNERS'
+      - 'LICENSE'
+      - 'Makefile'
+  push:
+    branches:
+      - main
+      - release/v[0-9]+.x
+      - release/v[0-9]+.[0-9]+.[0-9]+
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
+      - '.gitignore'
+      - 'CODEOWNERS'
+      - 'LICENSE'
+env:
+  GOARCH: amd64
+  CGO_ENABLED: 0
+  SETUP_GO_VERSION: '^1.20'
+  YQ_VERSION: v4.25.1
+  K3S_MIN_VERSION_TAG: v1.31.10-k3s1
+  E2E_CI: true
+  REPO: rancher
+  APISERVER_PORT: 8001
+  DEFAULT_SLEEP_TIMEOUT_SECONDS: 10
+  KUBECTL_WAIT_TIMEOUT: 300s
+  DEBUG: ${{ github.event.inputs.debug || false }}
+  CLUSTER_NAME: 'e2e-ci-kuberlr-kubectl'
+
+permissions:
+  contents: write
+
+jobs:
+  e2e-kuberlr-kubectl:
+    strategy:
+      matrix:
+        arch:
+          - x64
+          - arm64
+    runs-on: ${{ github.repository == 'rancher/kuberlr-kubectl' && format('runs-on,image=ubuntu22-full-{1},runner=4cpu-linux-{1},run-id={0}', github.run_id, matrix.arch) || 'ubuntu-latest' }}
+    steps:
+      -
+        # Add support for more platforms with QEMU (optional)
+        # https://github.com/docker/setup-qemu-action
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3
+      -
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
+        with:
+          fetch-depth: 0
+      - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5
+        with:
+          go-version: '>=1.20.0'
+      - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4
+      - name : Install helm
+        uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+      - name: Check if yq is installed
+        id: check_yq
+        run: |
+          if ! command -v yq &> /dev/null; then
+            echo "yq not found, installing..."
+            echo "::set-output name=install_yq::true"
+          else
+            echo "yq is already installed"
+            YQ_BIN=$(which yq)
+            echo "::set-output name=install_yq::false"
+            echo "::set-output name=yq_path::$YQ_BIN"
+          fi
+      - name : Install YQ
+        if: steps.check_yq.outputs.install_yq == 'true'
+        run: |
+          sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${{ matrix.arch == 'x64' && 'amd64' || matrix.arch  }} -O /usr/bin/yq && sudo chmod +x /usr/bin/yq;
+      - name : Export image version
+        run : |
+          source ./scripts/version
+          echo REPO=$REPO >> $GITHUB_ENV
+          echo IMAGE=$IMAGE >> $GITHUB_ENV
+          echo TAG=$TAG >> $GITHUB_ENV
+          echo FULL_IMAGE=$FULL_IMAGE >> $GITHUB_ENV
+
+      - name: Set K3S_VERSION
+        run: echo "K3S_VERSION=${{ inputs.k3s_version || env.K3S_MIN_VERSION_TAG }}" >> $GITHUB_ENV
+      -
+        name: Perform pre-e2e image build
+        run: |
+          make package;
+          make package-helm;
+      -
+        name : Install k3d
+        run : ./.github/workflows/e2e/scripts/install-k3d.sh
+      -
+        name : Setup k3d cluster
+        run : ./.github/workflows/e2e/scripts/setup-cluster.sh
+      -
+        name: Import Images Into k3d
+        run: |
+          k3d image import ${FULL_IMAGE} -c $CLUSTER_NAME;
+      -
+        name: Setup kubectl context
+        run: |
+          kubectl config use-context "k3d-$CLUSTER_NAME";
+      -
+        name: Install Kuberlr-Kubectl
+        run: ./.github/workflows/e2e/scripts/install-ci-chart.sh;
+      - 
+        name: Check if Kuberlr-Kubectl is up
+        run: ./.github/workflows/e2e/scripts/validate-ci-chart.sh;
+
+      - 
+        name: Uninstall Kuberlr-Kubectl
+        run: ./.github/workflows/e2e/scripts/uninstall-ci-chart.sh;
+      - name: Generate artifacts on failure
+        if: failure()
+        run: ./.github/workflows/e2e/scripts/generate-artifacts.sh;
+      - name: Upload logs and manifests on failure
+        if: failure()
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4
+        with:
+          name: artifacts-${{ matrix.arch }}-${{ inputs.k3s_version || env.K3S_MIN_VERSION_TAG }}
+          path: artifacts/
+          retention-days: 1
+      - 
+        name: Delete k3d cluster
+        if: always()
+        run: k3d cluster delete $CLUSTER_NAME

.github/workflows/e2e/scripts/cluster-args.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -e
+set -x
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+case "${KUBERNETES_DISTRIBUTION_TYPE}" in
+"k3s")
+    cluster_args=""
+    ;;
+"rke")
+    cluster_args=""
+    ;;
+"rke2")
+    cluster_args=""
+    ;;
+*)
+    echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown"
+    exit 1
+esac

.github/workflows/e2e/scripts/entry

@@ -0,0 +1,37 @@
+#!/bin/bash
+set -e
+set -x
+
+E2E_SCRIPTS_ROOT=$(realpath $(dirname $0))
+GIT_ROOT=$(realpath "$E2E_SCRIPTS_ROOT/../../../..")
+
+cd $GIT_ROOT
+source "$GIT_ROOT/scripts/version"
+
+DEFAULT_SLEEP_TIMEOUT_SECONDS=${DEFAULT_SLEEP_TIMEOUT_SECONDS:-10}
+KUBECTL_WAIT_TIMEOUT=${KUBECTL_WAIT_TIMEOUT:-120s}
+
+if [[ ${DEBUG} == "true" ]]; then
+    echo "Enabling DEBUG mode..."
+    set -x
+fi
+
+if [[ "${E2E_CI}" == "true" ]]; then
+    KUBERNETES_DISTRIBUTION_TYPE=k3s
+fi
+
+if [[ -n ${RANCHER_URL} ]] && [[ -n ${RANCHER_CLUSTER} ]] && [[ -n ${RANCHER_TOKEN} ]]; then
+    if [[ -n ${RANCHER_CLUSTER} && ${RANCHER_CLUSTER} != "local" ]]; then 
+        API_SERVER_URL=${RANCHER_URL}/k8s/clusters/${RANCHER_CLUSTER}
+    else
+        API_SERVER_URL=${RANCHER_URL}
+    fi 
+    API_SERVER_CURL_AUTH_HEADERS="-k -H 'Authorization: Bearer ${RANCHER_TOKEN}'"
+    RANCHER_CLUSTER_HELM_ARGS="--set global.cattle.url=${RANCHER_URL} --set global.cattle.clusterId=${RANCHER_CLUSTER}"
+else
+    kubectl proxy --port=${APISERVER_PORT:-8001} 2>/dev/null &
+    API_SERVER_URL=http://localhost:${APISERVER_PORT:-8001}
+    sleep 5
+fi
+
+RANCHER_HELM_ARGS="${RANCHER_CLUSTER_HELM_ARGS} ${RANCHER_MONITORING_VERSION_HELM_ARGS}"

.github/workflows/e2e/scripts/generate-artifacts.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+set -e
+set -x
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+case "${KUBERNETES_DISTRIBUTION_TYPE}" in
+"k3s")
+    components=(
+        "k3s-server"
+    )
+    ;;
+"rke")
+    components=(
+        "kube-controller-manager"
+        "kube-scheduler"
+        "kube-proxy"
+        "kube-etcd"
+    )
+    ;;
+"rke2")
+    components=(
+        "kube-controller-manager"
+        "kube-scheduler"
+        "kube-proxy"
+        "kube-etcd"
+    )
+    ;;
+*)
+    echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown"
+    exit 1
+esac
+
+ARTIFACT_DIRECTORY=artifacts
+DESCRIBE_DIRECTORY=${ARTIFACT_DIRECTORY}/described
+MANIFEST_DIRECTORY=${ARTIFACT_DIRECTORY}/manifests
+LOG_DIRECTORY=${ARTIFACT_DIRECTORY}/logs
+
+# Manifests
+mkdir -p ${MANIFEST_DIRECTORY}
+mkdir -p ${MANIFEST_DIRECTORY}/daemonsets
+mkdir -p ${MANIFEST_DIRECTORY}/deployments
+mkdir -p ${MANIFEST_DIRECTORY}/jobs
+mkdir -p ${MANIFEST_DIRECTORY}/statefulsets
+mkdir -p ${MANIFEST_DIRECTORY}/pods
+
+kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true
+kubectl get services -A > ${MANIFEST_DIRECTORY}/services-list.txt || true
+
+## cattle-ci-system ns manifests
+kubectl get daemonset -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/daemonsets/cattle-ci-system.yaml || true
+kubectl get deployment -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-ci-system.yaml || true
+kubectl get job -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/jobs/cattle-ci-system.yaml || true
+kubectl get statefulset -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-ci-system.yaml || true
+kubectl get pods -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-ci-system.yaml || true
+
+# Logs
+
+## Rancher logs
+mkdir -p ${LOG_DIRECTORY}/rancher
+
+kubectl logs deployment/rancher-webhook -n cattle-system > ${LOG_DIRECTORY}/rancher/rancher_webhook.log || true
+kubectl logs deployment/cattle-cluster-agent -n cattle-system > ${LOG_DIRECTORY}/rancher/cluster_agent.log || true
+kubectl logs deployment/system-upgrade-controller -n cattle-system > ${LOG_DIRECTORY}/rancher/upgrade_controller.log || true
+
+mkdir -p ${LOG_DIRECTORY}/rancher-monitoring
+
+## Rancher Kubectl-Kuberlr
+# TODO: decide what is worth collecting

.github/workflows/e2e/scripts/install-ci-chart.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+set -x
+
+source $(dirname $0)/entry
+source $(dirname $0)/cluster-args.sh
+
+cd $(dirname $0)/../../../..
+
+helm upgrade --install --create-namespace -n cattle-ci-system rancher-kuberlr-kubectl-debug \
+  --set global.kubectl.image.repository=${REPO:-rancher}/kuberlr-kubectl \
+  --set global.kubectl.image.tag=${TAG:-dev} \
+  ${cluster_args} \
+  ${RANCHER_HELM_ARGS} ./build/charts/kuberlr-kubectl-test
+
+echo "PASS: Kuberlr-Kubectl CI chart has been installed"

.github/workflows/e2e/scripts/install-k3d.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+set -x
+
+K3D_URL=https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh
+DEFAULT_K3D_VERSION=v5.7.4
+
+install_k3d(){
+  local k3dVersion=${K3D_VERSION:-${DEFAULT_K3D_VERSION}}
+  echo -e "Downloading k3d@${k3dVersion} see: ${K3D_URL}"
+  curl --silent --fail ${K3D_URL} | TAG=${k3dVersion} bash
+}
+
+install_k3d
+
+k3d version

.github/workflows/e2e/scripts/setup-cluster.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -e
+
+source ./scripts/version
+K3S_MIN_VERSION_TAG=v1.31.10-k3s1
+export K3S_VERSION=${K3S_VERSION:-$K3S_MIN_VERSION_TAG}
+
+if [ -z "$CLUSTER_NAME" ]; then
+    echo "CLUSTER_NAME must be specified when setting up a cluster"
+    exit 1
+fi
+
+if [ -z "$K3S_VERSION" ]; then
+  echo "K3S_VERSION must be specified when setting up a cluster, use $(k3d version list k3s) to find valid versions"
+  exit 1
+fi
+
+# waits until all nodes are ready
+wait_for_nodes(){
+  timeout=120
+  start_time=$(date +%s)
+  echo "wait until all agents are ready"
+  while :
+  do
+    current_time=$(date +%s)
+    elapsed_time=$((current_time - start_time))
+    if [ $elapsed_time -ge $timeout ]; then
+        echo "Timeout reached, exiting..."
+        exit 1
+    fi
+
+    readyNodes=1
+    statusList=$(kubectl get nodes --no-headers | awk '{ print $2}')
+    # shellcheck disable=SC2162
+    while read status
+    do
+      current_time=$(date +%s)
+      elapsed_time=$((current_time - start_time))
+      if [ $elapsed_time -ge $timeout ]; then
+          echo "Timeout reached, exiting..."
+          exit 1
+      fi
+      if [ "$status" == "NotReady" ] || [ "$status" == "" ]
+      then
+        readyNodes=0
+        break
+      fi
+    done <<< "$(echo -e  "$statusList")"
+    # all nodes are ready; exit
+    if [[ $readyNodes == 1 ]]
+    then
+      break
+    fi
+    sleep 1
+  done
+}
+
+k3d cluster delete "$CLUSTER_NAME" || true
+k3d cluster create "$CLUSTER_NAME" --image "docker.io/rancher/k3s:${K3S_VERSION}"
+
+wait_for_nodes
+
+echo "$CLUSTER_NAME ready"
+
+kubectl cluster-info --context "k3d-${CLUSTER_NAME}"
+kubectl config use-context "k3d-${CLUSTER_NAME}"
+kubectl get nodes -o wide

.github/workflows/e2e/scripts/uninstall-ci-chart.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -e
+set -x
+
+source $(dirname $0)/entry
+
+cd $(dirname $0)/../../../..
+
+helm uninstall --wait -n cattle-ci-system rancher-kuberlr-kubectl-debug
+
+echo "PASS: Kuberlr-Kubectl Shell has been uninstalled"