Skip to content
This repository was archived by the owner on Jan 29, 2025. It is now read-only.

Commit feb5025

Browse files
Muldoon, Killianmartinkennelly
authored and
togashidm
committed
Add scripts to run Kind for end-to-end testing
This change adds scripts to be used in automated and manual tests of Telemetry Aware Scheduling. The scripts set up a kind cluster install the custom metrics pipeline and installs and configures TAS to be ready to schedule workloads. Co-authored-by: martinkennelly <martin.kennelly@intel.com>
1 parent 560c799 commit feb5025

File tree

8 files changed

+245
-0
lines changed

8 files changed

+245
-0
lines changed

‎.github/scripts/e2e_cleanup.sh

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
#!/bin/bash
2+
# Remove any test artifacts created by tests
3+
set -o errexit
4+
5+
root="$(dirname "$0")/../../"
6+
tmp_dir="${root:?}/tmp"
7+
8+
echo "removing '${tmp_dir}'"
9+
rm -rf --preserve-root "${tmp_dir:?}"

‎.github/scripts/e2e_get_tools.sh

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
#!/bin/bash
2+
set -o errexit
3+
4+
root="$(dirname "$0")/../../"
5+
VERSION="v0.10.0"
6+
KIND_BINARY_URL="https://github.com/kubernetes-sigs/kind/releases/download/${VERSION}/kind-$(uname)-amd64"
7+
K8_STABLE_RELEASE_URL="https://storage.googleapis.com/kubernetes-release/release/stable.txt"
8+
9+
HELM_STABLE_RELEASE_URL="https://get.helm.sh/helm-v3.5.4-linux-amd64.tar.gz"
10+
11+
if [ ! -d "${root:?}/tmp/bin" ]; then
12+
mkdir -p "${root:?}/tmp/bin"
13+
fi
14+
15+
echo "retrieving kind"
16+
curl --max-time 10 --retry 10 --retry-delay 5 --retry-max-time 60 -Lo "${root}/tmp/bin/kind" "${KIND_BINARY_URL}"
17+
chmod +x "${root}/tmp/bin/kind"
18+
19+
echo "retrieving kubectl"
20+
curl -Lo "${root}/tmp/bin/kubectl" "https://storage.googleapis.com/kubernetes-release/release/$(curl -s ${K8_STABLE_RELEASE_URL})/tmp/bin/linux/amd64/kubectl"
21+
chmod +x "${root}/tmp/bin/kubectl"
22+
23+
24+
echo "retrieving helm"
25+
curl --max-time 10 --retry 10 --retry-delay 5 --retry-max-time 60 -Lo "${root}/tmp/helm.tar.gz" "${HELM_STABLE_RELEASE_URL}"
26+
tar -zxvf "${root}/tmp/helm.tar.gz" && mv linux-amd64/helm "${root}/tmp/bin/helm" && rm -rf linux-amd64
27+
chmod +x "${root}/tmp/bin/helm"

‎.github/scripts/e2e_setup_cluster.sh

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
#!/bin/bash
2+
set -eo pipefail
3+
4+
root="$(dirname "$0")/../../"
5+
export PATH="${PATH}:${root:?}/bin:${root:?}/tmp/bin"
6+
RETRY_MAX=10
7+
INTERVAL=10
8+
TIMEOUT=300
9+
APP_NAME="tasextender"
10+
APP_DOCKER_TAG="${APP_NAME}:latest"
11+
K8_ADDITIONS_PATH="${root}/.github/scripts/policies"
12+
TMP_DIR="${root}/tmp"
13+
CNIS_DAEMONSET_URL="https://raw.githubusercontent.com/intel/multus-cni/master/e2e/cni-install.yml"
14+
CNIS_NAME="cni-plugins"
15+
16+
# create cluster CA and policy for Kubernetes Scheduler
17+
# CA cert & key along with will be mounted to control plane
18+
# path /etc/kubernetes/pki. Kubeadm will utilise generated CA cert/key as root
19+
# Kubernetes CA. Cert for scheduler/TAS will be signed by this CA
20+
generate_k8_scheduler_config_data() {
21+
mkdir -p "${TMP_DIR}"
22+
mount_dir="$(mktemp -q -p "${TMP_DIR}" -d -t tas-e2e-k8-XXXXXXXX)"
23+
cp "${K8_ADDITIONS_PATH}/policy.yaml" "${mount_dir}/"
24+
}
25+
26+
create_cluster() {
27+
[ -z "${mount_dir}" ] && echo "### no mount directory set" && exit 1
28+
# deploy cluster with kind
29+
cat <<EOF | kind create cluster --config=-
30+
kind: Cluster
31+
apiVersion: kind.x-k8s.io/v1alpha4
32+
kubeadmConfigPatches:
33+
- |
34+
kind: ClusterConfiguration
35+
scheduler:
36+
dnsPolicy: ClusterFirstWithHostNet
37+
extraArgs:
38+
config: /etc/kubernetes/policy/policy.yaml
39+
extraVolumes:
40+
- name: kubeconfig
41+
hostPath: /etc/kubernetes/scheduler.conf
42+
mountPath: /etc/kubernetes/scheduler.conf
43+
- name: certs
44+
hostPath: /etc/kubernetes/pki/
45+
mountPath: /etc/kubernetes/pki/
46+
- name: schedulerconfig
47+
hostPath: /etc/kubernetes/policy/policy.yaml
48+
mountPath: /etc/kubernetes/policy/policy.yaml
49+
nodes:
50+
- role: control-plane
51+
extraMounts:
52+
- hostPath: "${mount_dir:?}"
53+
containerPath: "/etc/kubernetes/policy/"
54+
- role: worker
55+
extraMounts:
56+
- hostPath: "${mount_dir}/node1"
57+
containerPath: "/tmp/node-metrics/node1.prom"
58+
propagation: HostToContainer
59+
- role: worker
60+
extraMounts:
61+
- hostPath: "${mount_dir}/node2"
62+
containerPath: "/tmp/node-metrics/node2.prom"
63+
propagation: HostToContainer
64+
- role: worker
65+
extraMounts:
66+
- hostPath: "${mount_dir}/node3"
67+
containerPath: "/tmp/node-metrics/node3.prom"
68+
propagation: HostToContainer
69+
70+
EOF
71+
}
72+
73+
retry() {
74+
local status=0
75+
local retries=${RETRY_MAX:=5}
76+
local delay=${INTERVAL:=5}
77+
local to=${TIMEOUT:=20}
78+
cmd="$*"
79+
80+
while [ $retries -gt 0 ]
81+
do
82+
status=0
83+
timeout $to bash -c "echo $cmd && $cmd" || status=$?
84+
if [ $status -eq 0 ]; then
85+
break;
86+
fi
87+
echo "Exit code: '$status'. Sleeping '$delay' seconds before retrying"
88+
sleep "$delay"
89+
retries=$((retries-1))
90+
done
91+
return $status
92+
}
93+
94+
check_requirements() {
95+
for cmd in docker kind openssl kubectl base64; do
96+
if ! command -v "$cmd" &> /dev/null; then
97+
echo "$cmd is not available"
98+
exit 1
99+
fi
100+
done
101+
}
102+
103+
echo "## checking requirements"
104+
check_requirements
105+
# generate K8 API server CA key/cert and supporting files for mTLS with NRI
106+
echo "## generating K8s scheduler config"
107+
generate_k8_scheduler_config_data
108+
109+
110+
echo "## copy node metrics files to mount path"
111+
cp "${K8_ADDITIONS_PATH}/node1" "${mount_dir}"
112+
cp "${K8_ADDITIONS_PATH}/node2" "${mount_dir}"
113+
cp "${K8_ADDITIONS_PATH}/node3" "${mount_dir}"
114+
115+
116+
echo "## start Kind cluster with precreated CA key/cert"
117+
create_cluster
118+
119+
120+
121+
kubectl create namespace monitoring;
122+
helm install node-exporter "${root}/telemetry-aware-scheduling/deploy/charts/prometheus_node_exporter_helm_chart/";
123+
124+
125+
helm install prometheus "${root}/telemetry-aware-scheduling/deploy/charts/prometheus_helm_chart/";
126+
docker exec kind-control-plane mkdir -p /tmp/node-metrics/;
127+
128+
openssl req -x509 -sha256 -new -nodes -days 365 -newkey rsa:2048 -keyout "${TMP_DIR}/serving-ca.key" -out "${TMP_DIR}/serving-ca.crt" -subj "/CN=ca";
129+
kubectl create namespace custom-metrics ;kubectl -n custom-metrics create secret tls cm-adapter-serving-certs --cert="${TMP_DIR}/serving-ca.crt" --key="${TMP_DIR}/serving-ca.key";
130+
helm install prometheus-adapter "${root}/telemetry-aware-scheduling/deploy/charts/prometheus_custom_metrics_helm_chart/"
131+
132+
echo "## build TAS"
133+
retry make build
134+
retry make image
135+
echo "## load TAS image into Kind"
136+
kind load docker-image "${APP_DOCKER_TAG}"
137+
138+
echo "## config for kube-scheduler dns"
139+
docker cp kind-control-plane:/etc/kubernetes/manifests/kube-scheduler.yaml "${TMP_DIR}/kube-scheduler.yaml"
140+
141+
sed -e "/spec/a\\
142+
dnsPolicy: ClusterFirstWithHostNet" "${TMP_DIR}/kube-scheduler.yaml" -i
143+
144+
145+
docker cp "${TMP_DIR}/kube-scheduler.yaml" kind-control-plane:/etc/kubernetes/manifests/kube-scheduler.yaml
146+
echo "## install coreDNS"
147+
kubectl -n kube-system wait --for=condition=available deploy/coredns --timeout=300s
148+
echo "## install CNIs"
149+
retry kubectl create -f "${CNIS_DAEMONSET_URL}"
150+
retry kubectl -n kube-system wait --for=condition=ready -l name="${CNIS_NAME}" pod --timeout=300s
151+
152+
153+
mkdir "${mount_dir}/certs"
154+
docker cp kind-control-plane:/etc/kubernetes/pki/ca.crt "${mount_dir}/certs/client.crt"
155+
docker cp kind-control-plane:/etc/kubernetes/pki/ca.key "${mount_dir}/certs/client.key"
156+
157+
158+
kubectl create secret tls extender-secret --cert "${mount_dir}/certs/client.crt" --key "${mount_dir}/certs/client.key"
159+
kubectl apply -f "${root}/telemetry-aware-scheduling/deploy/"

‎.github/scripts/e2e_teardown_cluster.sh

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/bin/bash
2+
# Teardown Kind cluster
3+
4+
root="$(dirname "$0")/../../"
5+
export PATH="${PATH}:${root:?}/tmp/bin"
6+
7+
if ! command -v kind &> /dev/null; then
8+
echo "kind is not available. Run 'make e2e' first"
9+
exit 1
10+
fi
11+
12+
kind delete cluster

‎.github/scripts/policies/node1

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
node_filter1_metric 10
2+
node_filter2_metric 0
3+
node_prioritize1_metric 1000

‎.github/scripts/policies/node2

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
node_filter1_metric 20
2+
node_filter2_metric 0
3+
node_prioritize1_metric 9999
4+
node_deschedule1_metric 9

‎.github/scripts/policies/node3

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
node_filter1_metric 10
2+
node_filter2_metric 0
3+
node_prioritize1_metric 0
4+
5+

‎.github/scripts/policies/policy.yaml

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
{
2+
"kind" : "KubeSchedulerConfiguration",
3+
"apiVersion" : "kubescheduler.config.k8s.io/v1beta1",
4+
"clientConnection": {
5+
"kubeconfig": "/etc/kubernetes/scheduler.conf"
6+
},
7+
"extenders" : [
8+
{
9+
"urlPrefix": "https://tas-service.default.svc.cluster.local:9001",
10+
"prioritizeVerb": "scheduler/prioritize",
11+
"filterVerb": "scheduler/filter",
12+
"weight": 100,
13+
"managedResources": [
14+
{
15+
"name": "telemetry/scheduling",
16+
"ignoredByScheduler": true
17+
}
18+
],
19+
"tlsConfig": {
20+
"insecure": true,
21+
"certFile": "/etc/kubernetes/pki/ca.crt",
22+
"keyFile" : "/etc/kubernetes/pki/ca.key"
23+
}
24+
}
25+
]
26+
}

0 commit comments

Comments
 (0)