diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml new file mode 100644 index 0000000..77ccc1f --- /dev/null +++ b/.github/workflows/e2e-ci.yaml @@ -0,0 +1,144 @@ +name: "E2E CI" + +on: + workflow_call: + workflow_dispatch: + inputs: + debug: + description: "Enable debug logs" + required: false + default: "false" + k3s_version: + description: "Version of k3s to use for the underlying cluster, should exist in https://hub.docker.com/r/rancher/k3s/tags" + required: false + pull_request: + paths-ignore: + - 'docs/**' + - '*.md' + - '.gitignore' + - 'CODEOWNERS' + - 'LICENSE' + - 'Makefile' + push: + branches: + - main + - release/v[0-9]+.x + - release/v[0-9]+.[0-9]+.[0-9]+ + paths-ignore: + - 'docs/**' + - '*.md' + - '.gitignore' + - 'CODEOWNERS' + - 'LICENSE' +env: + GOARCH: amd64 + CGO_ENABLED: 0 + SETUP_GO_VERSION: '^1.20' + YQ_VERSION: v4.25.1 + K3S_MIN_VERSION_TAG: v1.31.10-k3s1 + E2E_CI: true + REPO: rancher + APISERVER_PORT: 8001 + DEFAULT_SLEEP_TIMEOUT_SECONDS: 10 + KUBECTL_WAIT_TIMEOUT: 300s + DEBUG: ${{ github.event.inputs.debug || false }} + CLUSTER_NAME: 'e2e-ci-kuberlr-kubectl' + +permissions: + contents: write + +jobs: + e2e-kuberlr-kubectl: + strategy: + matrix: + arch: + - x64 + - arm64 + runs-on: ${{ github.repository == 'rancher/kuberlr-kubectl' && format('runs-on,image=ubuntu22-full-{1},runner=4cpu-linux-{1},run-id={0}', github.run_id, matrix.arch) || 'ubuntu-latest' }} + steps: + - + # Add support for more platforms with QEMU (optional) + # https://github.com/docker/setup-qemu-action + name: Set up QEMU + uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3 + - + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4 + with: + fetch-depth: 0 + - uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5 + with: + go-version: '>=1.20.0' + - uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4 + - name : Install helm + uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Check if yq is installed + id: check_yq + run: | + if ! command -v yq &> /dev/null; then + echo "yq not found, installing..." + echo "::set-output name=install_yq::true" + else + echo "yq is already installed" + YQ_BIN=$(which yq) + echo "::set-output name=install_yq::false" + echo "::set-output name=yq_path::$YQ_BIN" + fi + - name : Install YQ + if: steps.check_yq.outputs.install_yq == 'true' + run: | + sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${{ matrix.arch == 'x64' && 'amd64' || matrix.arch }} -O /usr/bin/yq && sudo chmod +x /usr/bin/yq; + - name : Export image version + run : | + source ./scripts/version + echo REPO=$REPO >> $GITHUB_ENV + echo IMAGE=$IMAGE >> $GITHUB_ENV + echo TAG=$TAG >> $GITHUB_ENV + echo FULL_IMAGE=$FULL_IMAGE >> $GITHUB_ENV + + - name: Set K3S_VERSION + run: echo "K3S_VERSION=${{ inputs.k3s_version || env.K3S_MIN_VERSION_TAG }}" >> $GITHUB_ENV + - + name: Perform pre-e2e image build + run: | + make package; + make package-helm; + - + name : Install k3d + run : ./.github/workflows/e2e/scripts/install-k3d.sh + - + name : Setup k3d cluster + run : ./.github/workflows/e2e/scripts/setup-cluster.sh + - + name: Import Images Into k3d + run: | + k3d image import ${FULL_IMAGE} -c $CLUSTER_NAME; + - + name: Setup kubectl context + run: | + kubectl config use-context "k3d-$CLUSTER_NAME"; + - + name: Install Kuberlr-Kubectl + run: ./.github/workflows/e2e/scripts/install-ci-chart.sh; + - + name: Check if Kuberlr-Kubectl is up + run: ./.github/workflows/e2e/scripts/validate-ci-chart.sh; + + - + name: Uninstall Kuberlr-Kubectl + run: ./.github/workflows/e2e/scripts/uninstall-ci-chart.sh; + - name: Generate artifacts on failure + if: failure() + run: ./.github/workflows/e2e/scripts/generate-artifacts.sh; + - name: Upload logs and manifests on failure + if: failure() + uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4 + with: + name: artifacts-${{ matrix.arch }}-${{ inputs.k3s_version || env.K3S_MIN_VERSION_TAG }} + path: artifacts/ + retention-days: 1 + - + name: Delete k3d cluster + if: always() + run: k3d cluster delete $CLUSTER_NAME diff --git a/.github/workflows/e2e/scripts/cluster-args.sh b/.github/workflows/e2e/scripts/cluster-args.sh new file mode 100644 index 0000000..ddddadc --- /dev/null +++ b/.github/workflows/e2e/scripts/cluster-args.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e +set -x + +source $(dirname $0)/entry + +cd $(dirname $0)/../../../.. + +case "${KUBERNETES_DISTRIBUTION_TYPE}" in +"k3s") + cluster_args="" + ;; +"rke") + cluster_args="" + ;; +"rke2") + cluster_args="" + ;; +*) + echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown" + exit 1 +esac \ No newline at end of file diff --git a/.github/workflows/e2e/scripts/entry b/.github/workflows/e2e/scripts/entry new file mode 100755 index 0000000..d8f6ffe --- /dev/null +++ b/.github/workflows/e2e/scripts/entry @@ -0,0 +1,37 @@ +#!/bin/bash +set -e +set -x + +E2E_SCRIPTS_ROOT=$(realpath $(dirname $0)) +GIT_ROOT=$(realpath "$E2E_SCRIPTS_ROOT/../../../..") + +cd $GIT_ROOT +source "$GIT_ROOT/scripts/version" + +DEFAULT_SLEEP_TIMEOUT_SECONDS=${DEFAULT_SLEEP_TIMEOUT_SECONDS:-10} +KUBECTL_WAIT_TIMEOUT=${KUBECTL_WAIT_TIMEOUT:-120s} + +if [[ ${DEBUG} == "true" ]]; then + echo "Enabling DEBUG mode..." + set -x +fi + +if [[ "${E2E_CI}" == "true" ]]; then + KUBERNETES_DISTRIBUTION_TYPE=k3s +fi + +if [[ -n ${RANCHER_URL} ]] && [[ -n ${RANCHER_CLUSTER} ]] && [[ -n ${RANCHER_TOKEN} ]]; then + if [[ -n ${RANCHER_CLUSTER} && ${RANCHER_CLUSTER} != "local" ]]; then + API_SERVER_URL=${RANCHER_URL}/k8s/clusters/${RANCHER_CLUSTER} + else + API_SERVER_URL=${RANCHER_URL} + fi + API_SERVER_CURL_AUTH_HEADERS="-k -H 'Authorization: Bearer ${RANCHER_TOKEN}'" + RANCHER_CLUSTER_HELM_ARGS="--set global.cattle.url=${RANCHER_URL} --set global.cattle.clusterId=${RANCHER_CLUSTER}" +else + kubectl proxy --port=${APISERVER_PORT:-8001} 2>/dev/null & + API_SERVER_URL=http://localhost:${APISERVER_PORT:-8001} + sleep 5 +fi + +RANCHER_HELM_ARGS="${RANCHER_CLUSTER_HELM_ARGS} ${RANCHER_MONITORING_VERSION_HELM_ARGS}" \ No newline at end of file diff --git a/.github/workflows/e2e/scripts/generate-artifacts.sh b/.github/workflows/e2e/scripts/generate-artifacts.sh new file mode 100755 index 0000000..58bc77d --- /dev/null +++ b/.github/workflows/e2e/scripts/generate-artifacts.sh @@ -0,0 +1,71 @@ +#!/bin/bash +set -e +set -x + +source $(dirname $0)/entry + +cd $(dirname $0)/../../../.. + +case "${KUBERNETES_DISTRIBUTION_TYPE}" in +"k3s") + components=( + "k3s-server" + ) + ;; +"rke") + components=( + "kube-controller-manager" + "kube-scheduler" + "kube-proxy" + "kube-etcd" + ) + ;; +"rke2") + components=( + "kube-controller-manager" + "kube-scheduler" + "kube-proxy" + "kube-etcd" + ) + ;; +*) + echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown" + exit 1 +esac + +ARTIFACT_DIRECTORY=artifacts +DESCRIBE_DIRECTORY=${ARTIFACT_DIRECTORY}/described +MANIFEST_DIRECTORY=${ARTIFACT_DIRECTORY}/manifests +LOG_DIRECTORY=${ARTIFACT_DIRECTORY}/logs + +# Manifests +mkdir -p ${MANIFEST_DIRECTORY} +mkdir -p ${MANIFEST_DIRECTORY}/daemonsets +mkdir -p ${MANIFEST_DIRECTORY}/deployments +mkdir -p ${MANIFEST_DIRECTORY}/jobs +mkdir -p ${MANIFEST_DIRECTORY}/statefulsets +mkdir -p ${MANIFEST_DIRECTORY}/pods + +kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true +kubectl get services -A > ${MANIFEST_DIRECTORY}/services-list.txt || true + +## cattle-ci-system ns manifests +kubectl get daemonset -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/daemonsets/cattle-ci-system.yaml || true +kubectl get deployment -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-ci-system.yaml || true +kubectl get job -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/jobs/cattle-ci-system.yaml || true +kubectl get statefulset -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-ci-system.yaml || true +kubectl get pods -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-ci-system.yaml || true + +# Logs + +## Rancher logs +mkdir -p ${LOG_DIRECTORY}/rancher + +kubectl logs deployment/rancher-webhook -n cattle-system > ${LOG_DIRECTORY}/rancher/rancher_webhook.log || true +kubectl logs deployment/cattle-cluster-agent -n cattle-system > ${LOG_DIRECTORY}/rancher/cluster_agent.log || true +kubectl logs deployment/system-upgrade-controller -n cattle-system > ${LOG_DIRECTORY}/rancher/upgrade_controller.log || true + +mkdir -p ${LOG_DIRECTORY}/rancher-monitoring + +## Rancher Kubectl-Kuberlr +# TODO: decide what is worth collecting diff --git a/.github/workflows/e2e/scripts/install-ci-chart.sh b/.github/workflows/e2e/scripts/install-ci-chart.sh new file mode 100755 index 0000000..8b458c5 --- /dev/null +++ b/.github/workflows/e2e/scripts/install-ci-chart.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e +set -x + +source $(dirname $0)/entry +source $(dirname $0)/cluster-args.sh + +cd $(dirname $0)/../../../.. + +helm upgrade --install --create-namespace -n cattle-ci-system rancher-kuberlr-kubectl-debug \ + --set global.kubectl.image.repository=${REPO:-rancher}/kuberlr-kubectl \ + --set global.kubectl.image.tag=${TAG:-dev} \ + ${cluster_args} \ + ${RANCHER_HELM_ARGS} ./build/charts/kuberlr-kubectl-test + +echo "PASS: Kuberlr-Kubectl CI chart has been installed" diff --git a/.github/workflows/e2e/scripts/install-k3d.sh b/.github/workflows/e2e/scripts/install-k3d.sh new file mode 100755 index 0000000..1aa640e --- /dev/null +++ b/.github/workflows/e2e/scripts/install-k3d.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e +set -x + +K3D_URL=https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh +DEFAULT_K3D_VERSION=v5.7.4 + +install_k3d(){ + local k3dVersion=${K3D_VERSION:-${DEFAULT_K3D_VERSION}} + echo -e "Downloading k3d@${k3dVersion} see: ${K3D_URL}" + curl --silent --fail ${K3D_URL} | TAG=${k3dVersion} bash +} + +install_k3d + +k3d version \ No newline at end of file diff --git a/.github/workflows/e2e/scripts/setup-cluster.sh b/.github/workflows/e2e/scripts/setup-cluster.sh new file mode 100755 index 0000000..4a2c719 --- /dev/null +++ b/.github/workflows/e2e/scripts/setup-cluster.sh @@ -0,0 +1,68 @@ +#!/bin/bash + +set -e + +source ./scripts/version +K3S_MIN_VERSION_TAG=v1.31.10-k3s1 +export K3S_VERSION=${K3S_VERSION:-$K3S_MIN_VERSION_TAG} + +if [ -z "$CLUSTER_NAME" ]; then + echo "CLUSTER_NAME must be specified when setting up a cluster" + exit 1 +fi + +if [ -z "$K3S_VERSION" ]; then + echo "K3S_VERSION must be specified when setting up a cluster, use $(k3d version list k3s) to find valid versions" + exit 1 +fi + +# waits until all nodes are ready +wait_for_nodes(){ + timeout=120 + start_time=$(date +%s) + echo "wait until all agents are ready" + while : + do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ $elapsed_time -ge $timeout ]; then + echo "Timeout reached, exiting..." + exit 1 + fi + + readyNodes=1 + statusList=$(kubectl get nodes --no-headers | awk '{ print $2}') + # shellcheck disable=SC2162 + while read status + do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ $elapsed_time -ge $timeout ]; then + echo "Timeout reached, exiting..." + exit 1 + fi + if [ "$status" == "NotReady" ] || [ "$status" == "" ] + then + readyNodes=0 + break + fi + done <<< "$(echo -e "$statusList")" + # all nodes are ready; exit + if [[ $readyNodes == 1 ]] + then + break + fi + sleep 1 + done +} + +k3d cluster delete "$CLUSTER_NAME" || true +k3d cluster create "$CLUSTER_NAME" --image "docker.io/rancher/k3s:${K3S_VERSION}" + +wait_for_nodes + +echo "$CLUSTER_NAME ready" + +kubectl cluster-info --context "k3d-${CLUSTER_NAME}" +kubectl config use-context "k3d-${CLUSTER_NAME}" +kubectl get nodes -o wide \ No newline at end of file diff --git a/.github/workflows/e2e/scripts/uninstall-ci-chart.sh b/.github/workflows/e2e/scripts/uninstall-ci-chart.sh new file mode 100755 index 0000000..e3d9868 --- /dev/null +++ b/.github/workflows/e2e/scripts/uninstall-ci-chart.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +set -x + +source $(dirname $0)/entry + +cd $(dirname $0)/../../../.. + +helm uninstall --wait -n cattle-ci-system rancher-kuberlr-kubectl-debug + +echo "PASS: Kuberlr-Kubectl Shell has been uninstalled" diff --git a/.github/workflows/e2e/scripts/validate-ci-chart.sh b/.github/workflows/e2e/scripts/validate-ci-chart.sh new file mode 100755 index 0000000..dd1ea11 --- /dev/null +++ b/.github/workflows/e2e/scripts/validate-ci-chart.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e +set -x + +source $(dirname $0)/entry + +cd $(dirname $0)/../../../.. + +if ! kubectl -n cattle-ci-system rollout status deployment kuberlr-kubectl-shell --timeout="${KUBECTL_WAIT_TIMEOUT}"; then + echo "ERROR: Kuberlr-Kubectl Shell did not roll out" + exit 1 +fi + +echo "PASS: Kuberlr-Kubectl Shell is up and running" + +if ! kubectl -n cattle-ci-system rollout status deployment kuberlr-kubectl-action --timeout="${KUBECTL_WAIT_TIMEOUT}"; then + echo "ERROR: Kuberlr-Kubectl Shell did not roll out" + exit 1 +fi + +echo "PASS: Kuberlr-Kubectl Shell is up and running" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a79baac..620fed2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,14 @@ jobs: - name: Check out repository code uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + - name : Make test helm chart + run: TAG=$TAG_NAME make package-helm + - name: Add test helm chart to release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release upload "$TAG_NAME" "./build/charts/rancher-kubectl-test-${TAG_NAME}.tgz" + - name: Load Secrets from Vault uses: rancher-eio/read-vault-secrets@main with: diff --git a/.gitignore b/.gitignore index 3e51a61..9628afb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ versions.txt new-versions.txt /image_arch_test *.oci -kubectl-versions.txt.log \ No newline at end of file +kubectl-versions.txt.log +kubeconfig.yaml \ No newline at end of file diff --git a/Makefile b/Makefile index db36b6f..8c7e557 100644 --- a/Makefile +++ b/Makefile @@ -55,4 +55,4 @@ ifdef DIRTY @git --no-pager status @git --no-pager diff @exit 1 -endif \ No newline at end of file +endif diff --git a/charts/kuberlr-kubectl-test/Chart.yaml b/charts/kuberlr-kubectl-test/Chart.yaml new file mode 100644 index 0000000..0670188 --- /dev/null +++ b/charts/kuberlr-kubectl-test/Chart.yaml @@ -0,0 +1,20 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Rancher Kubectl Testing + catalog.cattle.io/kube-version: '>= 1.26.0-0' + catalog.cattle.io/namespace: cattle-testing + catalog.cattle.io/os: linux + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/rancher-version: '>= 2.9.0-0' + catalog.cattle.io/release-name: rancher-kubectl-test + catalog.cattle.io/upstream-version: 999 +apiVersion: v2 +appVersion: 999 +description: Provides ability to test the kubectl image from Rancher. +keywords: + - applications + - infrastructure + - testing +kubeVersion: '>= 1.26.0-0' +name: rancher-kubectl-test +version: 999 \ No newline at end of file diff --git a/charts/kuberlr-kubectl-test/README.md b/charts/kuberlr-kubectl-test/README.md new file mode 100644 index 0000000..9d56cdf --- /dev/null +++ b/charts/kuberlr-kubectl-test/README.md @@ -0,0 +1,29 @@ +# Kubectl Test Chart +This chart facilitates testing of RC versions of `rancher/kuberlr-kubectl` image. + +Specifically this allows us to create, test and QA an RC version of the image without needing to involve dependent charts. +The goal being that we shouldn't need to "mix processes" and utilize another project for testing this project. + +## What does this test for? +To correctly provide a validation w/o using consuming charts, we need to ensure this chart does the same type of stuff. +Additionally, we should ensure that when it does those actions it will create loud and obvious errors when failure happens. + +Some examples of scenarios this should cover: +- Post upgrade jobs (like BRO, Monitoring), +- Creating a `ServiceAccount` (with appropriate bindings) and using that for upgrade task (like BRO), +- Storing upgrade scripts in a `ConfigMap` and then running them on upgrade (like Monitoring) + +As we find more specific use-cases within Rancher that existing scenarios do not cover, we should create issues to track adding new tests. + +## How does (or will) this work? + +This chart will be packaged as a release artifact with each GitHub release. +In turn every tag going forward will have a direct 1:1 chart that can be used for testing and QA of that tag. + +Eng will create a new RC and upon success of the action, they will inform QA (via the related issue) that the RC is ready for testing. +Once QA picks up the issue for testing they will fetch the debug/QA chart from the release and install via CLI. +Then they will subsequently "upgrade" to the same version to trigger any upgrade specific jobs. + +After each phase QA will be able to verify the resources and jobs in the testing namespace. +As long as they do not observe any errors - just as they would expect for a production chart - the RC has passed. +Then once the `rancher/kuberlr-kubectl` tag has been un-RC'd any consuming charts can update to the new stable tag. \ No newline at end of file diff --git a/charts/kuberlr-kubectl-test/templates/_helpers.tpl b/charts/kuberlr-kubectl-test/templates/_helpers.tpl new file mode 100644 index 0000000..7bc0182 --- /dev/null +++ b/charts/kuberlr-kubectl-test/templates/_helpers.tpl @@ -0,0 +1,97 @@ +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- else -}} +{{- "" -}} +{{- end -}} +{{- end -}} + +{{/* +The image to use +*/}} +{{- define "kubectl.image" -}} +{{- $temp_registry := (include "system_default_registry" .) }} +{{- if $temp_registry }} +{{- printf "%s%s:%s" $temp_registry .Values.global.kubectl.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.global.kubectl.image.tag) }} +{{- else if .Values.global.imageRegistry }} +{{- printf "%s/%s:%s" .Values.global.imageRegistry .Values.global.kubectl.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.global.kubectl.image.tag) }} +{{- else }} +{{- printf "%s:%s" .Values.global.kubectl.image.repository (default (printf "v%s" .Chart.AppVersion) .Values.global.kubectl.image.tag) }} +{{- end }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +The components in this chart create additional resources that expand the longest created name strings. +The longest name that gets created adds and extra 37 characters, so truncation should be 63-35=26. +*/}} +{{- define "kubectlTest.fullname" -}} +{{- if .Values.fullnameOverride -}} +{{- .Values.fullnameOverride | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 26 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 26 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "kubectlTest.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "kubectlTest.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* +Common labels +*/}} +{{- define "kubectlTest.labels" -}} +helm.sh/chart: {{ include "kubectlTest.chart" . }} +{{ include "kubectlTest.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "kubectlTest.selectorLabels" -}} +app.kubernetes.io/name: {{ include "kubectlTest.fullname" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "kubectlTest.serviceAccountName" -}} +{{ include "kubectlTest.fullname" . }} +{{- end }} + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} \ No newline at end of file diff --git a/charts/kuberlr-kubectl-test/templates/deployment.yaml b/charts/kuberlr-kubectl-test/templates/deployment.yaml new file mode 100644 index 0000000..598b3d8 --- /dev/null +++ b/charts/kuberlr-kubectl-test/templates/deployment.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kuberlr-kubectl-shell +spec: + replicas: 1 + selector: + matchLabels: + app: kuberlr-kubectl-shell + template: + metadata: + labels: + app: kuberlr-kubectl-shell + spec: + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: kuberlr-kubectl-shell-container + image: {{ template "kubectl.image" . }} + imagePullPolicy: {{ .Values.global.kubectl.image.pullPolicy | default "Always" }} + command: ["/bin/bash", "-c", "sleep infinity"] # This keeps the container running + {{- if and .Values.global.kubectl (hasKey .Values.global.kubectl "kuberlrAllowDownload") }} + env: + - name: KUBERLR_ALLOWDOWNLOAD + value: {{ default .Values.global.kubectl.kuberlrAllowDownload false }} + {{- end }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kuberlr-kubectl-action +spec: + replicas: 1 + selector: + matchLabels: + app: kuberlr-kubectl-action + template: + metadata: + labels: + app: kuberlr-kubectl-action + spec: + securityContext: + runAsNonRoot: false + runAsUser: 0 + containers: + - name: kuberlr-kubectl-action-container + image: {{ template "kubectl.image" . }} + imagePullPolicy: {{ .Values.global.kubectl.image.pullPolicy | default "Always" }} + command: ["/bin/bash", "-c", " + set -e + echo 'Check kuberlr bins...' && + kuberlr bins && + echo \"kubectl bin linked at: `which kubectl`\" && + ls -lah `which kubectl` && + echo 'Starting main loop...' && + sleep infinity + "] # This keeps the container running + {{- if and .Values.global.kubectl (hasKey .Values.global.kubectl "kuberlrAllowDownload") }} + env: + - name: KUBERLR_ALLOWDOWNLOAD + value: {{ default .Values.global.kubectl.kuberlrAllowDownload false }} + {{- end }} \ No newline at end of file diff --git a/charts/kuberlr-kubectl-test/values.yaml b/charts/kuberlr-kubectl-test/values.yaml new file mode 100644 index 0000000..025aae6 --- /dev/null +++ b/charts/kuberlr-kubectl-test/values.yaml @@ -0,0 +1,12 @@ +global: + cattle: + systemDefaultRegistry: "" + kubectl: + image: + repository: rancher/kuberlr-kubectl + tag: '' + pullPolicy: "IfNotPresent" + # As long as kubectl image is using `rancher/kuberlr-kubectl` it can use kuberlr features. + # kuberlrAllowDownload is disabled by default for most compatibility with air-gaps + # When enabled it will allow kuberlr to download a compatible `kubectl` with the target cluster. + # kuberlrAllowDownload: false \ No newline at end of file diff --git a/scripts/ci b/scripts/ci index 650b2b2..4252545 100755 --- a/scripts/ci +++ b/scripts/ci @@ -8,9 +8,9 @@ cd "$SCRIPTS_DIR" ./version -if [[ -f "$PROJECT_DIR/multiarch-image.oci" ]]; then +if [[ -f "$PROJECT_DIR/ci/multiarch-image.oci" ]]; then # Remove old OCI file - rm "$PROJECT_DIR/multiarch-image.oci" + rm "$PROJECT_DIR/ci/multiarch-image.oci" echo "Removed old CI OCI file" fi diff --git a/scripts/local-e2e b/scripts/local-e2e new file mode 100755 index 0000000..8e63fd4 --- /dev/null +++ b/scripts/local-e2e @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +set -e + +DONT_CLEAN=${DONT_CLEAN:-false} + +header() { + local text="$1" + local width=53 # Adjust this for desired total width + local padding=$(( (width - ${#text}) / 2 )) # Calculate padding for centering + + printf '%*s\n' "$width" | tr ' ' '-' + printf '%*s%s%*s\n' "$padding" "" "$text" "$padding" "" + printf '%*s\n' "$width" | tr ' ' '-' +} + +cleanupTest() { + header "CLEANUP" + echo "Cleaning up before exit..." + if k3d cluster list $CLUSTER_NAME 2> /dev/null; then + k3d cluster delete $CLUSTER_NAME + fi +} + +onExit() { + if [[ "$?" -eq 0 ]] || [ "$DONT_CLEAN" == true ]; then + exit 0 + fi + + cleanupTest +} +trap onExit EXIT + +source "$(dirname "$0")/version" +K3S_MIN_VERSION_TAG=v1.31.10-k3s1 + +cd "$(dirname "$0")/.." + +# Setup CI specific Vars +export CLUSTER_NAME='e2e-ci-kuberlr-kubectl' +export E2E_CI=true +export K3S_VERSION=${K3S_VERSION:-$K3S_MIN_VERSION_TAG} + +if k3d cluster list $CLUSTER_NAME 2> /dev/null; then + echo "The test cluster '$CLUSTER_NAME' already exists for some reason" + echo "Either manually delete the tests cluster, or pick a new cluser name to use." + exit 0 +fi + +set +e +KUBE_PROXY_PIDS=$(ps aux|grep -v grep|grep kubectl|grep proxy) +set -e +if [ "$KUBE_PROXY_PIDS" != "" ]; then + echo "$KUBE_PROXY_PIDS" | while IFS= read -r kubeProxy; do + kill -9 "$(echo "$kubeProxy" | awk '{print $2}')" + done +fi + +header "Starting E2E tests for Kuberlr-Kubectl@$TAG" + +# Pre build images +header "Building Kuberlr-Kubectl images for testing" +REPO=${REPO} TAG=${TAG} make build-image; +REPO=${REPO} TAG=${TAG} make package-helm; +header "Image built ${REPO}/kuberlr-kubectl:${TAG}" +export TAG + +# Install k3d +which k3d > /dev/null 2>&1 +if [ "$?" -eq 1 ]; then + echo "Found k3d not installed setting up now..." + ./.github/workflows/e2e/scripts/install-k3d.sh +fi + +# Setup k3d cluster +header "Setting up E2E cluster" +./.github/workflows/e2e/scripts/setup-cluster.sh + +# Import Images Into k3d +header "Importing images to k3d" +k3d image import ${REPO}/kuberlr-kubectl:${TAG} -c $CLUSTER_NAME; + +# Setup kubectl context +kubectl config use-context "k3d-$CLUSTER_NAME"; + +## TODO: install & verify any prereqs to test + +# Install Kuberlr-Kubectl +header "Installing kuberlr-kubectl CI chart" +./.github/workflows/e2e/scripts/install-ci-chart.sh; + +# Check if Kuberlr-Kubectl is up +header "Verifying kuberlr-kubectl CI is UP" +./.github/workflows/e2e/scripts/validate-ci-chart.sh; + +## TODO: steps to verify the images? + +### ALL LOGIC ABOVE THIS +if [ "$DONT_CLEAN" == true ]; then + header "Local e2e testing was a SUCCESS" + header "Exiting early, to leave cluster for testing" + exit; +fi + +# Uninstall Kuberlr-Kubectl +header "Uninstall kuberlr-kubectl CI chart" +./.github/workflows/e2e/scripts/uninstall-ci-chart.sh; + +header "Local e2e testing was a SUCCESS" +cleanupTest \ No newline at end of file diff --git a/scripts/package b/scripts/package new file mode 100755 index 0000000..b09a424 --- /dev/null +++ b/scripts/package @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +set -e + +source $(dirname $0)/version + +cd $(dirname $0)/.. + +echo "Starting \`${IMAGE_NAME}\` image packaging:"; + +echo "Building \`${FULL_IMAGE}\` from \`package/Dockerfile\`:"; + +if [[ ${USE_DOCKER_BUILDX} -eq 1 ]]; then + docker buildx build --platform linux/amd64,linux/arm64 -f package/Dockerfile --build-arg RANCHER_PROJECT_MONITORING=$RANCHER_PROJECT_MONITORING -t "${FULL_IMAGE}" . +else + docker build -f package/Dockerfile --build-arg RANCHER_PROJECT_MONITORING=$RANCHER_PROJECT_MONITORING -t "${FULL_IMAGE}" . +fi + +echo "Completed building ${FULL_IMAGE} container image" diff --git a/scripts/package-ci b/scripts/package-ci index 2c4f90c..671319d 100755 --- a/scripts/package-ci +++ b/scripts/package-ci @@ -1,14 +1,17 @@ #!/bin/bash set -e -set -x echo "Starting package script..." SCRIPTS_DIR=$(realpath $(dirname $0)) source "$SCRIPTS_DIR/version" +source "$SCRIPTS_DIR/util-ci-helpers" GIT_ROOT=$(dirname $SCRIPTS_DIR) cd "$GIT_ROOT" -REPO=${REPO} TAG=${TAG} make build-validate \ No newline at end of file +assert_file_not_exists ci/multiarch-image.oci "Must run clean up before this script, or run via ci script" +REPO=${REPO} TAG=${TAG} make build-validate +assert_dir_exists ci +assert_file_exists ci/multiarch-image.oci \ No newline at end of file diff --git a/scripts/package-helm b/scripts/package-helm new file mode 100755 index 0000000..148de4c --- /dev/null +++ b/scripts/package-helm @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +if ! hash helm 2>/dev/null; then + echo "Helm is not installed" + exit 1 +fi + +source "$(dirname "$0")/version" +source "$(dirname "$0")/util-chart" +#source "$(dirname "$0")/util-team-charts" + +cd "$(dirname "$0")/.." + +echo "Preparing Kuberlr-Kuebctl testing chart" +clean-old-charts +cp -rf charts build/ + +if [[ "$REPO" != "rancher" ]]; then + echo "Using custom repo: $REPO to build chart" + edit-chart ./build/charts/kuberlr-kubectl-test/Chart.yaml "${HELM_CHART_VERSION}" "${HELM_IMAGE_TAG}" "${REPO}" +else + edit-chart ./build/charts/kuberlr-kubectl-test/Chart.yaml "${HELM_CHART_VERSION}" "${HELM_IMAGE_TAG}" +fi +if ! package-chart ./build/charts/kuberlr-kubectl-test ./build/charts ; then + echo "package-chart failed..." + exit 1 +fi \ No newline at end of file diff --git a/scripts/util-chart b/scripts/util-chart new file mode 100644 index 0000000..351c36d --- /dev/null +++ b/scripts/util-chart @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +function is_gnu_sed(){ + sed --version >/dev/null 2>&1 +} + +function sed_i_wrapper(){ + if is_gnu_sed; then + $(which sed) "$@" + else + a=() + for b in "$@"; do + [[ $b == '-i' ]] && a=("${a[@]}" "$b" "") || a=("${a[@]}" "$b") + done + $(which sed) "${a[@]}" + fi +} + +function edit-chart() { + sed_i_wrapper -i \ + -e "s/^version:.*/version: ${2}/" \ + -e "s/^appVersion:.*/appVersion: ${3}/" \ + "${1}" + sed_i_wrapper -i \ + -e "s/tag: ''/tag: ${3}/" \ + "${1/Chart.yaml/values.yaml}" + + + if [ $# -ge 4 ]; then # Check if at least 4 arguments were passed + sed_i_wrapper -i \ + -e "s#repository: rancher/kuberlr-kubectl#repository: ${4}/kuberlr-kubectl#" \ + "${1/Chart.yaml/values.yaml}" + fi +} + +function package-chart() { + helm package --debug "${1:-"./build/charts/kubectl-test"}" -d "${2:-"./build/charts"}" +} + +function clean-old-charts() { + if [[ ! -d "./build/charts" ]]; then + mkdir -p "./build/charts" + fi +} \ No newline at end of file diff --git a/scripts/util-ci-helpers b/scripts/util-ci-helpers new file mode 100644 index 0000000..11c52e7 --- /dev/null +++ b/scripts/util-ci-helpers @@ -0,0 +1,101 @@ +#!/usr/bin/env bash + +# Function to record a test result +record_test_result() { + TEST_COUNT=$((TEST_COUNT + 1)) + local test_name="$1" + local success="$2" # true or false + local optional_message="$3" # The optional message + + if [[ "$success" == "true" ]]; then + echo " ✅ Assertion PASS: ${test_name}" + else + echo " ❌ Assertion FAIL: ${test_name}" + if [[ -n "$optional_message" ]]; then # Check if message is not empty + echo " Info: ${optional_message}" + fi + FAIL_COUNT=$((FAIL_COUNT + 1)) + fi +} + +# Assertion: Check if a file exists +# Usage: assert_file_exists "file_path" [optional_message] +assert_file_exists() { + local file_path="$1" + local test_name="File exists: ${file_path}" + local optional_message="$2" # Capture the optional message + + if [[ -f "${file_path}" ]]; then + record_test_result "${test_name}" "true" "$optional_message" + return 0 # Success + else + record_test_result "${test_name}" "false" "$optional_message" + return 1 # Failure + fi +} + +# Assertion: Check if a file does NOT exist +# Usage: assert_file_not_exists "file_path" [optional_message] +assert_file_not_exists() { + local file_path="$1" + local test_name="File does not exist: ${file_path}" + local optional_message="$2" # Capture the optional message + + if [[ ! -f "${file_path}" ]]; then + record_test_result "${test_name}" "true" "$optional_message" + return 0 # Success + else + record_test_result "${test_name}" "false" "$optional_message" + return 1 # Failure + fi +} + +# Assertion: Check if a directory exists +# Usage: assert_dir_exists "dir_path" [optional_message] +assert_dir_exists() { + local dir_path="$1" + local test_name="Directory exists: ${dir_path}" + local optional_message="$2" # Capture the optional message + + if [[ -d "${dir_path}" ]]; then + record_test_result "${test_name}" "true" "$optional_message" + return 0 # Success + else + record_test_result "${test_name}" "false" "$optional_message" + return 1 # Failure + fi +} + +# Assertion: Run a command and check its exit status +# Usage: assert_command_success "command string" [optional_message] +assert_command_success() { + local command_to_run="$1" # Takes the first argument as the command string + local optional_message="$2" # The second argument is the optional message + local test_name="Command '${command_to_run}' succeeds" + + # Execute the command and capture its exit status + if eval "${command_to_run}" > /dev/null 2>&1; then + record_test_result "${test_name}" "true" "$optional_message" + return 0 # Success + else + record_test_result "${test_name}" "false" "$optional_message" + return 1 # Failure + fi +} + +# Assertion: Run a command and check for a non-zero exit status (failure) +# Usage: assert_command_failure "command string" [optional_message] +assert_command_failure() { + local command_to_run="$1" # Takes the first argument as the command string + local optional_message="$2" # The second argument is the optional message + local test_name="Command '${command_to_run}' fails" + + # Execute the command and capture its exit status + if ! eval "${command_to_run}" > /dev/null 2>&1; then + record_test_result "${test_name}" "true" "$optional_message" + return 0 # Success + else + record_test_result "${test_name}" "false" "$optional_message" + return 1 # Failure + fi +} \ No newline at end of file diff --git a/scripts/version b/scripts/version index 32dd4dc..2ab6b81 100755 --- a/scripts/version +++ b/scripts/version @@ -37,7 +37,9 @@ if echo "$TAG" | grep -q dirty; then HELM_IMAGE_TAG=$TAG HELM_CHART_VERSION=${HELM_CHART_VERSION_DEV:-${HELM_IMAGE_TAG/v/}} fi -IMAGE=${IMAGE:-"$REPO/kuberlr-kubectl:${TAG}"} +IMAGE=${IMAGE:-"kuberlr-kubectl"} +IMAGE_NAME=${IMAGE_NAME:-"${REPO}/${IMAGE}"} +FULL_IMAGE=${FULL_IMAGE:-"${IMAGE_NAME}:${TAG}"} if [[ "$1" == "--short" && -n "$2" ]]; then var_name="$2" @@ -50,7 +52,8 @@ function print_version_debug() { echo "SUFFIX: $SUFFIX"; echo "HELM_IMAGE_TAG: $HELM_IMAGE_TAG"; echo "HELM_CHART_VERSION: $HELM_CHART_VERSION"; - echo "REPO: $REPO; TAG: $TAG"; - echo "IMAGE: $IMAGE"; + echo "REPO: $REPO; IMAGE: $IMAGE; TAG: $TAG"; + echo "IMAGE_NAME: $IMAGE_NAME" + echo "FULL_IMAGE: $FULL_IMAGE"; } if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then print_version_debug "$1"; fi \ No newline at end of file