Skip to content

Commit 9192c66

Browse files
mallardduckmallardduck
committed
initial e2e script setup
1 parent fc7a564 commit 9192c66

File tree

21 files changed

+565
-167
lines changed

21 files changed

+565
-167
lines changed

‎.github/workflows/e2e-ci.yaml‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
name: "E2E CI"
2+
3+
on:
4+
workflow_call:
5+
workflow_dispatch:
6+
inputs:
7+
debug:
8+
description: "Enable debug logs"
9+
required: false
10+
default: "false"
11+
k3s_version:
12+
description: "Version of k3s to use for the underlying cluster, should exist in https://hub.docker.com/r/rancher/k3s/tags"
13+
required: false
14+
pull_request:
15+
paths-ignore:
16+
- 'docs/**'
17+
- '*.md'
18+
- '.gitignore'
19+
- 'CODEOWNERS'
20+
- 'LICENSE'
21+
- 'Makefile'
22+
push:
23+
branches:
24+
- main
25+
- release/v[0-9]+.x
26+
- release/v[0-9]+.[0-9]+.[0-9]+
27+
paths-ignore:
28+
- 'docs/**'
29+
- '*.md'
30+
- '.gitignore'
31+
- 'CODEOWNERS'
32+
- 'LICENSE'
33+
env:
34+
GOARCH: amd64
35+
CGO_ENABLED: 0
36+
SETUP_GO_VERSION: '^1.20'
37+
YQ_VERSION: v4.25.1
38+
K3S_MIN_VERSION_TAG: v1.31.10-k3s1
39+
E2E_CI: true
40+
REPO: rancher
41+
APISERVER_PORT: 8001
42+
DEFAULT_SLEEP_TIMEOUT_SECONDS: 10
43+
KUBECTL_WAIT_TIMEOUT: 300s
44+
DEBUG: ${{ github.event.inputs.debug || false }}
45+
CLUSTER_NAME: 'e2e-ci-kuberlr-kubectl'
46+
47+
permissions:
48+
contents: write
49+
50+
jobs:
51+
e2e-kuberlr-kubectl:
52+
strategy:
53+
matrix:
54+
arch:
55+
- x64
56+
- arm64
57+
runs-on: ${{ github.repository == 'rancher/kuberlr-kubectl' && format('runs-on,image=ubuntu22-full-{1},runner=4cpu-linux-{1},run-id={0}', github.run_id, matrix.arch) || 'ubuntu-latest' }}
58+
steps:
59+
-
60+
# Add support for more platforms with QEMU (optional)
61+
# https://github.com/docker/setup-qemu-action
62+
name: Set up QEMU
63+
uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3
64+
-
65+
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4
66+
with:
67+
fetch-depth: 0
68+
- uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5
69+
with:
70+
go-version: '>=1.20.0'
71+
- uses: azure/setup-kubectl@3e0aec4d80787158d308d7b364cb1b702e7feb7f # v4
72+
- name : Install helm
73+
uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4
74+
with:
75+
token: ${{ secrets.GITHUB_TOKEN }}
76+
- name: Check if yq is installed
77+
id: check_yq
78+
run: |
79+
if ! command -v yq &> /dev/null; then
80+
echo "yq not found, installing..."
81+
echo "::set-output name=install_yq::true"
82+
else
83+
echo "yq is already installed"
84+
YQ_BIN=$(which yq)
85+
echo "::set-output name=install_yq::false"
86+
echo "::set-output name=yq_path::$YQ_BIN"
87+
fi
88+
- name : Install YQ
89+
if: steps.check_yq.outputs.install_yq == 'true'
90+
run: |
91+
sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${{ matrix.arch == 'x64' && 'amd64' || matrix.arch }} -O /usr/bin/yq && sudo chmod +x /usr/bin/yq;
92+
- name : Export image version
93+
run : |
94+
source ./scripts/version
95+
echo REPO=$REPO >> $GITHUB_ENV
96+
echo IMAGE=$IMAGE >> $GITHUB_ENV
97+
echo TAG=$TAG >> $GITHUB_ENV
98+
echo FULL_IMAGE=$FULL_IMAGE >> $GITHUB_ENV
99+
100+
- name: Set K3S_VERSION
101+
run: echo "K3S_VERSION=${{ inputs.k3s_version || env.K3S_MIN_VERSION_TAG }}" >> $GITHUB_ENV
102+
-
103+
name: Perform pre-e2e image build
104+
run: |
105+
make package;
106+
make package-helm;
107+
-
108+
name : Install k3d
109+
run : ./.github/workflows/e2e/scripts/install-k3d.sh
110+
-
111+
name : Setup k3d cluster
112+
run : ./.github/workflows/e2e/scripts/setup-cluster.sh
113+
-
114+
name: Import Images Into k3d
115+
run: |
116+
k3d image import ${FULL_IMAGE} -c $CLUSTER_NAME;
117+
-
118+
name: Setup kubectl context
119+
run: |
120+
kubectl config use-context "k3d-$CLUSTER_NAME";
121+
-
122+
name: Install Kuberlr-Kubectl
123+
run: ./.github/workflows/e2e/scripts/install-ci-chart.sh;
124+
-
125+
name: Check if Kuberlr-Kubectl is up
126+
run: ./.github/workflows/e2e/scripts/validate-ci-chart.sh;
127+
128+
-
129+
name: Uninstall Kuberlr-Kubectl
130+
run: ./.github/workflows/e2e/scripts/uninstall-ci-chart.sh;
131+
- name: Generate artifacts on failure
132+
if: failure()
133+
run: ./.github/workflows/e2e/scripts/generate-artifacts.sh;
134+
- name: Upload logs and manifests on failure
135+
if: failure()
136+
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4
137+
with:
138+
name: artifacts-${{ matrix.arch }}-${{ inputs.k3s_version || env.K3S_MIN_VERSION_TAG }}
139+
path: artifacts/
140+
retention-days: 1
141+
-
142+
name: Delete k3d cluster
143+
if: always()
144+
run: k3d cluster delete e2e-ci-kuberlr-kubectl

‎.github/workflows/e2e/scripts/cluster-args.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
set -e
3+
set -x
4+
5+
source $(dirname $0)/entry
6+
7+
cd $(dirname $0)/../../../..
8+
9+
case "${KUBERNETES_DISTRIBUTION_TYPE}" in
10+
"k3s")
11+
cluster_args=""
12+
;;
13+
"rke")
14+
cluster_args=""
15+
;;
16+
"rke2")
17+
cluster_args=""
18+
;;
19+
*)
20+
echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown"
21+
exit 1
22+
esac

‎.github/workflows/e2e/scripts/entry‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/bash
2+
set -e
3+
set -x
4+
5+
E2E_SCRIPTS_ROOT=$(realpath $(dirname $0))
6+
GIT_ROOT=$(realpath "$E2E_SCRIPTS_ROOT/../../../..")
7+
8+
cd $GIT_ROOT
9+
source "$GIT_ROOT/scripts/version"
10+
11+
DEFAULT_SLEEP_TIMEOUT_SECONDS=${DEFAULT_SLEEP_TIMEOUT_SECONDS:-10}
12+
KUBECTL_WAIT_TIMEOUT=${KUBECTL_WAIT_TIMEOUT:-120s}
13+
14+
if [[ ${DEBUG} == "true" ]]; then
15+
echo "Enabling DEBUG mode..."
16+
set -x
17+
fi
18+
19+
if [[ "${E2E_CI}" == "true" ]]; then
20+
KUBERNETES_DISTRIBUTION_TYPE=k3s
21+
fi
22+
23+
if [[ -n ${RANCHER_URL} ]] && [[ -n ${RANCHER_CLUSTER} ]] && [[ -n ${RANCHER_TOKEN} ]]; then
24+
if [[ -n ${RANCHER_CLUSTER} && ${RANCHER_CLUSTER} != "local" ]]; then
25+
API_SERVER_URL=${RANCHER_URL}/k8s/clusters/${RANCHER_CLUSTER}
26+
else
27+
API_SERVER_URL=${RANCHER_URL}
28+
fi
29+
API_SERVER_CURL_AUTH_HEADERS="-k -H 'Authorization: Bearer ${RANCHER_TOKEN}'"
30+
RANCHER_CLUSTER_HELM_ARGS="--set global.cattle.url=${RANCHER_URL} --set global.cattle.clusterId=${RANCHER_CLUSTER}"
31+
else
32+
kubectl proxy --port=${APISERVER_PORT:-8001} 2>/dev/null &
33+
API_SERVER_URL=http://localhost:${APISERVER_PORT:-8001}
34+
sleep 5
35+
fi
36+
37+
RANCHER_HELM_ARGS="${RANCHER_CLUSTER_HELM_ARGS} ${RANCHER_MONITORING_VERSION_HELM_ARGS}"

‎.github/workflows/e2e/scripts/generate-artifacts.sh‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/bin/bash
2+
set -e
3+
set -x
4+
5+
source $(dirname $0)/entry
6+
7+
cd $(dirname $0)/../../../..
8+
9+
case "${KUBERNETES_DISTRIBUTION_TYPE}" in
10+
"k3s")
11+
components=(
12+
"k3s-server"
13+
)
14+
;;
15+
"rke")
16+
components=(
17+
"kube-controller-manager"
18+
"kube-scheduler"
19+
"kube-proxy"
20+
"kube-etcd"
21+
)
22+
;;
23+
"rke2")
24+
components=(
25+
"kube-controller-manager"
26+
"kube-scheduler"
27+
"kube-proxy"
28+
"kube-etcd"
29+
)
30+
;;
31+
*)
32+
echo "KUBERNETES_DISTRIBUTION_TYPE=${KUBERNETES_DISTRIBUTION_TYPE} is unknown"
33+
exit 1
34+
esac
35+
36+
ARTIFACT_DIRECTORY=artifacts
37+
DESCRIBE_DIRECTORY=${ARTIFACT_DIRECTORY}/described
38+
MANIFEST_DIRECTORY=${ARTIFACT_DIRECTORY}/manifests
39+
LOG_DIRECTORY=${ARTIFACT_DIRECTORY}/logs
40+
41+
# Manifests
42+
mkdir -p ${MANIFEST_DIRECTORY}
43+
mkdir -p ${MANIFEST_DIRECTORY}/daemonsets
44+
mkdir -p ${MANIFEST_DIRECTORY}/deployments
45+
mkdir -p ${MANIFEST_DIRECTORY}/jobs
46+
mkdir -p ${MANIFEST_DIRECTORY}/statefulsets
47+
mkdir -p ${MANIFEST_DIRECTORY}/pods
48+
49+
kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true
50+
kubectl get services -A > ${MANIFEST_DIRECTORY}/services-list.txt || true
51+
52+
## cattle-ci-system ns manifests
53+
kubectl get daemonset -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/daemonsets/cattle-ci-system.yaml || true
54+
kubectl get deployment -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-ci-system.yaml || true
55+
kubectl get job -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/jobs/cattle-ci-system.yaml || true
56+
kubectl get statefulset -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-ci-system.yaml || true
57+
kubectl get pods -n cattle-ci-system -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-ci-system.yaml || true
58+
59+
# Logs
60+
61+
## Rancher logs
62+
mkdir -p ${LOG_DIRECTORY}/rancher
63+
64+
kubectl logs deployment/rancher-webhook -n cattle-system > ${LOG_DIRECTORY}/rancher/rancher_webhook.log || true
65+
kubectl logs deployment/cattle-cluster-agent -n cattle-system > ${LOG_DIRECTORY}/rancher/cluster_agent.log || true
66+
kubectl logs deployment/system-upgrade-controller -n cattle-system > ${LOG_DIRECTORY}/rancher/upgrade_controller.log || true
67+
68+
mkdir -p ${LOG_DIRECTORY}/rancher-monitoring
69+
70+
## Rancher Kubectl-Kuberlr
71+
# TODO: decide what is worth collecting

‎.github/workflows/e2e/scripts/install-ci-chart.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
set -e
3+
set -x
4+
5+
source $(dirname $0)/entry
6+
source $(dirname $0)/cluster-args.sh
7+
8+
cd $(dirname $0)/../../../..
9+
10+
helm upgrade --install --create-namespace -n cattle-ci-system rancher-kuberlr-kubectl-debug \
11+
--set global.kubectl.image.repository=${REPO:-rancher}/kuberlr-kubectl \
12+
--set global.kubectl.image.tag=${TAG:-dev} \
13+
${cluster_args} \
14+
${RANCHER_HELM_ARGS} ./build/charts/kuberlr-kubectl-test
15+
16+
echo "PASS: Kuberlr-Kubectl CI chart has been installed"

‎.github/workflows/e2e/scripts/install-k3d.sh‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -x
5+
6+
K3D_URL=https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh
7+
DEFAULT_K3D_VERSION=v5.7.4
8+
9+
install_k3d(){
10+
local k3dVersion=${K3D_VERSION:-${DEFAULT_K3D_VERSION}}
11+
echo -e "Downloading k3d@${k3dVersion} see: ${K3D_URL}"
12+
curl --silent --fail ${K3D_URL} | TAG=${k3dVersion} bash
13+
}
14+
15+
install_k3d
16+
17+
k3d version

‎.github/workflows/e2e/scripts/setup-cluster.sh‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
#!/bin/bash
2+
3+
set -e
4+
5+
source ./scripts/version
6+
K3S_MIN_VERSION_TAG=v1.31.10-k3s1
7+
export K3S_VERSION=${K3S_VERSION:-$K3S_MIN_VERSION_TAG}
8+
9+
if [ -z "$CLUSTER_NAME" ]; then
10+
echo "CLUSTER_NAME must be specified when setting up a cluster"
11+
exit 1
12+
fi
13+
14+
if [ -z "$K3S_VERSION" ]; then
15+
echo "K3S_VERSION must be specified when setting up a cluster, use $(k3d version list k3s) to find valid versions"
16+
exit 1
17+
fi
18+
19+
# waits until all nodes are ready
20+
wait_for_nodes(){
21+
timeout=120
22+
start_time=$(date +%s)
23+
echo "wait until all agents are ready"
24+
while :
25+
do
26+
current_time=$(date +%s)
27+
elapsed_time=$((current_time - start_time))
28+
if [ $elapsed_time -ge $timeout ]; then
29+
echo "Timeout reached, exiting..."
30+
exit 1
31+
fi
32+
33+
readyNodes=1
34+
statusList=$(kubectl get nodes --no-headers | awk '{ print $2}')
35+
# shellcheck disable=SC2162
36+
while read status
37+
do
38+
current_time=$(date +%s)
39+
elapsed_time=$((current_time - start_time))
40+
if [ $elapsed_time -ge $timeout ]; then
41+
echo "Timeout reached, exiting..."
42+
exit 1
43+
fi
44+
if [ "$status" == "NotReady" ] || [ "$status" == "" ]
45+
then
46+
readyNodes=0
47+
break
48+
fi
49+
done <<< "$(echo -e "$statusList")"
50+
# all nodes are ready; exit
51+
if [[ $readyNodes == 1 ]]
52+
then
53+
break
54+
fi
55+
sleep 1
56+
done
57+
}
58+
59+
k3d cluster delete "$CLUSTER_NAME" || true
60+
k3d cluster create "$CLUSTER_NAME" --image "docker.io/rancher/k3s:${K3S_VERSION}"
61+
62+
wait_for_nodes
63+
64+
echo "$CLUSTER_NAME ready"
65+
66+
kubectl cluster-info --context "k3d-${CLUSTER_NAME}"
67+
kubectl config use-context "k3d-${CLUSTER_NAME}"
68+
kubectl get nodes -o wide

‎.github/workflows/e2e/scripts/uninstall-ci-chart.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
set -e
3+
set -x
4+
5+
source $(dirname $0)/entry
6+
7+
cd $(dirname $0)/../../../..
8+
9+
helm uninstall --wait -n cattle-ci-system rancher-kuberlr-kubectl-debug
10+
11+
echo "PASS: Kuberlr-Kubectl Shell has been uninstalled"

0 commit comments

Comments
 (0)