Skip to content
This repository was archived by the owner on Jan 29, 2025. It is now read-only.

Commit c757c09

Browse files
madalazartogashidm
madalazar
authored and
togashidm
committed
Configure TAS set-up to use kube-scheduler Configuration APIs for K8s v22 onwards
The purpose of this change is to set-up the TAS scheduler using kube-scheduler Configuration APIs instead of policies. kubernetes/kubernetes#105202 says that the policy API will be removed from K8s v23 onwards and suggested to customers to migrate to "component config." kube-scheduler Configuration API v1beta1 will be deprecated and v1beta3 of the same API will be introduced in K8sv23. We're expecting v1beta2 to still work in K8s v23 (via https://bit.ly/k8s123-enhancements)
1 parent 676f4b3 commit c757c09

File tree

2 files changed

+252
-27
lines changed

2 files changed

+252
-27
lines changed

‎telemetry-aware-scheduling/deploy/extender-configuration/configure-scheduler.sh

Lines changed: 234 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,41 +1,248 @@
11
#!/bin/sh
2-
## Set the location of the kube-scheduler manifest file. This is the location of the file under an ordinary Kubeadm set up.
3-
## Note this step will only work if the scheduler is running in the cluster. If it's running as a service/binary/ex-K8s container the flags will need to be applied separately.
2+
## Note this step will only work if the scheduler is running in the cluster. If it's running as a service/binary/ex-K8s
3+
## container the flags will need to be applied separately.
44

5+
is_test=false
6+
# [IMPORTANT] The script works for K8s clusters set-up via kubeadm with kube-scheduler.yaml configuration file
7+
# in /etc/kubernetes/manifest.
8+
# For a different cluster set-up, users shoud provide the path to their kube-scheduler.yaml file
59
MANIFEST_FILE=/etc/kubernetes/manifests/kube-scheduler.yaml
10+
scheduler_config_file_path=deploy/extender-configuration/scheduler-config.yaml
11+
scheduler_config_destination=/etc/kubernetes
612

7-
## Create the config map and the cluster role for the scheduler configuration. The cluster role is needed in Kubeadm to ensure the scheduler can access configmaps.
8-
CONFIG_MAP=scheduler-extender-configmap.yaml
9-
CLUSTER_ROLE=configmap-getter.yaml
13+
help() {
14+
echo "Usage: $(basename "$0") [-m PATH_TO_MANIFEST_FILE] [-f PATH_TO_CONFIGURATION_FILE] [-d CONFIGURATION_DESTINATION_FOLDER] [-th]" 2>&1
15+
echo 'Configure the Kubernetes scheduler using one or more of the parameters below. If not entered, the script will be using default values. '
16+
echo 'Please make sure the user used to run this script has read & write access to the files/directories mentioned below.'
17+
18+
echo ' -m PATH_TO_MANIFEST_FILE Specify the path to the Kubernetes manifest kube-scheduler.yaml file'
19+
echo ' If not provided, it will default to /etc/kubernetes/manifest/kube-scheduler.yaml'
20+
21+
echo ' -f PATH_TO_CONFIGURATION_FILE Specify the path to the kube scheduler configuration file. Required only from K8s v22 onwards.'
22+
echo ' If not provided, will default to scheduler-configuration (/deploy/extender-configuration)'
1023

11-
kubectl apply -f $CONFIG_MAP
12-
kubectl apply -f $CLUSTER_ROLE
13-
##Add clusterrole binding - default binding edit - to give kube-scheduler access to configmaps in kube-system.
14-
kubectl create clusterrolebinding scheduler-config-map --clusterrole=configmapgetter --user=system:kube-scheduler
24+
echo ' -d CONFIGURATION_DESTINATION_FOLDER Specify the destination folder for the kube scheduler config file. Required only from K8s v22 onwards.'
25+
echo ' If not provided, it will default to /etc/kubernetes/.'
26+
27+
echo ' -t Run script in test mode. It requires parameters to be provided to -m, -f, -d'
28+
echo ' In this state the script is intended to work with mock files and directories.'
29+
echo ' It will generate the expected configuration for the scheduler but it will'
30+
echo ' NOT configure the scheduler correctly.'
1531

16-
## Remove arguments from Kubernetes Scheduler file if they exist
17-
sed -i '/^ - --policy-configmap/d' $MANIFEST_FILE
18-
sed -i '/^ dnsPolicy: ClusterFirstWithHostNet/d' $MANIFEST_FILE
19-
sed -i '/certs/d' $MANIFEST_FILE
20-
sed -i '/name: certdir/d' $MANIFEST_FILE
21-
sed -i '/hostPath/d' $MANIFEST_FILE
32+
echo ' -h Show help menu.'
33+
}
2234

35+
###### PARSE THE CUSTOMER INPUT AND ASSIGN THE VALUES
36+
while getopts 'm:f:d:th' option; do
37+
case $option in
38+
h) # print help description
39+
help
40+
exit;;
41+
t) # start the script in test mode. We require parameters to be provided for -m, -f, -d in order to not accidentally
42+
# production configuration when in this mode
43+
if [ $# -lt 6 ]; then
44+
echo "Not enough input parameters were passed to the script. This requires -m, -f, -d to have values. Exiting..."
45+
exit
46+
fi
47+
is_test=true
48+
;;
49+
m)
50+
user_manifest_file_path=$OPTARG
51+
if [ -n "$user_manifest_file_path" ];
52+
then MANIFEST_FILE=$user_manifest_file_path
53+
fi
54+
# MANIFEST_FILE CHECKS
55+
if [ ! -f "$MANIFEST_FILE" ]; then
56+
echo "Critical error. $MANIFEST_FILE doesn't exist. Please check the cluster configuration. Exiting..."
57+
exit 1
58+
fi
59+
if ! test -r "$MANIFEST_FILE" || ! test -w "$MANIFEST_FILE"; then
60+
echo "Critical error. The user used to run this script doesn't have read or write access to $MANIFEST_FILE. Exiting..."
61+
exit 1
62+
fi
63+
;;
64+
f)
65+
# SCHEDULER CONFIG FILE CHECK
66+
user_scheduler_config_file_path=$OPTARG
67+
if [ -n "$user_scheduler_config_file_path" ]; then
68+
scheduler_config_file_path=$user_scheduler_config_file_path
69+
fi
70+
if ! test -r "$scheduler_config_file_path" || ! test -w "$scheduler_config_file_path"; then
71+
echo "Critical error. The user used to run this script doesn't have read or write access to $scheduler_config_file_path. Exiting..."
72+
exit 1
73+
fi
74+
;;
75+
d)
76+
# SCHEDULER CONFIG DESTINATION FOLDER CHECK
77+
user_scheduler_config_destination=$OPTARG
78+
if [ -n "$user_scheduler_config_destination" ]; then
79+
scheduler_config_destination=$user_scheduler_config_destination
80+
fi
81+
;;
82+
\?) # invalid option
83+
echo 'Error: Invalid option. Exiting...'
84+
exit;;
85+
esac
86+
done
87+
88+
echo "Manifest file is located at: $MANIFEST_FILE"
89+
echo "Scheduler config file is located at: $scheduler_config_file_path"
90+
echo "Scheduler config destination will be: $scheduler_config_destination"
91+
92+
####### DETERMINE THE VERSION OF SCHEDULER USED IN THE K8S CLUSTER
93+
scheduler_image_version_22=22
94+
# The scheduling configuration API is currently (K8s v22) in the v1beta2 version. From K8s v23 onwards the api
95+
# will have version v1.
96+
scheduler_config_api_versions_v1beta2="v1beta2"
97+
scheduler_image=$( grep "image:" "$MANIFEST_FILE" | cut -d '.' -f 4 )
98+
99+
if [ -z "$scheduler_image" ]; then
100+
echo "Unable to retrieve the scheduler image value from manifest file. We got: $scheduler_image. Exiting..."
101+
exit 1
102+
fi
103+
104+
echo "Version of the image used in the kube scheduler is: $scheduler_image"
105+
106+
####### CLEAN_UP MANIFEST FILE
107+
# In case the previous run of this script was partially successful or unsuccessful, we'd like to start from a clean
108+
# state, independent of any previous runs
109+
sed -i '/^ - --config/d' "$MANIFEST_FILE"
110+
sed -i '/^ - --policy-configmap/d' "$MANIFEST_FILE"
111+
sed -i '/^ - --policy-configmap-namespace/d' "$MANIFEST_FILE"
112+
sed -i '/^ dnsPolicy: ClusterFirstWithHostNet/d' "$MANIFEST_FILE"
113+
# retrieve the name of the scheduler config file
114+
scheduler_config_file=$(basename "$scheduler_config_file_path")
115+
# clean-up scheduler configuration
116+
sed -i '/hostPath/d' "$MANIFEST_FILE"
117+
sed -i "/$scheduler_config_file/d" "$MANIFEST_FILE"
118+
sed -i '/name: schedulerconfig/d' "$MANIFEST_FILE"
119+
# clean-up certs configuration
120+
sed -i '/certs/d' "$MANIFEST_FILE"
121+
sed -i '/name: certdir/d' "$MANIFEST_FILE"
122+
sed -i '/readOnly: true/d' "$MANIFEST_FILE"
123+
124+
####### SETTING UP NECESSARY CERTS
23125
## Copy client cert/key pair into kube-scheduler
24-
mkdir /etc/certs/
126+
if [ ! -d "/etc/certs/" ]; then
127+
echo "Will proceed to create /etc/certs/..."
128+
mkdir /etc/certs/
129+
fi
130+
131+
if [ ! -d "/etc/certs/" ]; then
132+
echo "Unable to successfully create the /etc/certs/ folder. Exiting..."
133+
exit 1
134+
fi
135+
25136
cp /etc/kubernetes/pki/ca.key /etc/certs/client.key
26137
cp /etc/kubernetes/pki/ca.crt /etc/certs/client.crt
27138

28-
## Add arguments to our kube-scheduler manifest. The arguments are:
29-
## 1) Policy configmap extender as arg to binary.
30-
## 2) Policy configmap namespace as arg to binary.
31-
## 3) dnsPolicy as part of Pod spec allowing access to kubernetes services.
32-
## 4) Set autorization certs
139+
####### INITIAL MANIFEST_FILE CHANGES
33140

34-
sed -e "/ - kube-scheduler/a\\
35-
- --policy-configmap=scheduler-extender-policy\n - --policy-configmap-namespace=kube-system" $MANIFEST_FILE -i
141+
## The arguments are:
142+
## 1) dnsPolicy as part of Pod spec allowing access to kubernetes services.
143+
## 2) Set authorization certs
144+
## 3) mount the necessary configuration files from the disk
36145
sed -e "/spec/a\\
37-
dnsPolicy: ClusterFirstWithHostNet" $MANIFEST_FILE -i
38-
sed -e "/ readOnly: true/a\\
39-
- mountPath: /host/certs\n name: certdir" $MANIFEST_FILE -i
146+
dnsPolicy: ClusterFirstWithHostNet" "$MANIFEST_FILE" -i
147+
sed -e "/ name: kubeconfig/a\\
148+
readOnly: true" "$MANIFEST_FILE" -i
149+
sed -e "/ volumeMounts:/a\\
150+
- mountPath: /host/certs\n name: certdir" "$MANIFEST_FILE" -i
40151
sed -e "/ volumes:/a\\
41-
- hostPath:\n path: /etc/certs\n name: certdir\n - hostPath:" $MANIFEST_FILE -i
152+
- hostPath:\n path: /etc/certs\n name: certdir\n - hostPath:" "$MANIFEST_FILE" -i
153+
154+
####### VERSION SPECIFIC MANIFEST_FILE CHANGES. These are necessary, but change according to the version of Kubernetes
155+
156+
## Before K8s v22 we will use the Policy API instead of the Scheduler Config API in order to setup the scheduler
157+
if [ "$scheduler_image" -lt $scheduler_image_version_22 ]; then
158+
echo "[IMPORTANT] Will proceed by using the Policy API to configure the scheduler extender. This API will be **DEPRECATED** from $scheduler_image_version_22 onwards"
159+
# Create the config map and the cluster role for the scheduler configuration.
160+
#The cluster role is needed in Kubeadm to ensure the scheduler can access configmaps.
161+
config_map=deploy/extender-configuration/scheduler-extender-configmap.yaml
162+
cluster_role=deploy/extender-configuration/configmap-getter.yaml
163+
164+
if ! $is_test ; then
165+
echo "Will proceed to generate the required K8s resources..."
166+
# check if the necessary files exist. If they don't the commands below will fail anyway
167+
if [ ! -f "$config_map" ]; then
168+
echo "Critical error: $config_map doesn't exist. Can't configure the scheduler for version $scheduler_image. Exiting..."
169+
exit 1
170+
fi
171+
if [ ! -f "$cluster_role" ]; then
172+
echo "Critical error: $cluster_role doesn't exist. Can't configure the scheduler for version $scheduler_image. Exiting..."
173+
exit 1
174+
fi
175+
176+
if ! kubectl apply -f $config_map; then
177+
echo "Unable to successfully apply $config_map. Will revert the change and exit."
178+
kubectl delete -f $config_map
179+
exit 1
180+
fi
181+
182+
if ! kubectl apply -f $cluster_role; then
183+
echo "Unable to successfully apply $cluster_role. Reverting the change... "
184+
kubectl delete -f $cluster_role
185+
echo " Reverting changes from $config_map as the resources from $cluster_role and $config_map are related."
186+
kubectl delete -f $config_map
187+
echo "Exit..."
188+
exit 1
189+
fi
190+
191+
# we aim to create this resource only once, as with the use of create the kubectl command below
192+
# will return an error if the resource in question already exists
193+
cluster_role_binding_result=$(kubectl get clusterrolebinding -A | grep scheduler-config-map)
194+
if [ -z "$cluster_role_binding_result" ]; then
195+
echo "Cluster role binding scheduler-config-map doesn't exist. Will proceed to create it..."
196+
#Add clusterrole binding - default binding edit - to give kube-scheduler access to configmaps in kube-system.
197+
kubectl create clusterrolebinding scheduler-config-map --clusterrole=configmapgetter --user=system:kube-scheduler
198+
fi
199+
fi
200+
## Add arguments to our kube-scheduler manifest. The arguments are:
201+
## 1) Policy configmap extender as arg to binary.
202+
## 2) Policy configmap namespace as arg to binary.
203+
sed -e "/ - kube-scheduler/a\\
204+
- --policy-configmap=scheduler-extender-policy\n - --policy-configmap-namespace=kube-system" "$MANIFEST_FILE" -i
205+
else
206+
echo "[IMPORTANT]Will proceed by using the kube-scheduler Configuration API instead for the scheduler extender. This API will be **USED** from $scheduler_image_version_22 onwards."
207+
currentKubeSchedulerApiVersion=$scheduler_config_api_versions_v1beta2
208+
echo "Kube Scheduler Configuration api version: $currentKubeSchedulerApiVersion"
209+
210+
if [ -z "$currentKubeSchedulerApiVersion" ]; then
211+
echo "Unable to determine the correct API version for Kube Scheduler Configuration. We got: $currentKubeSchedulerApiVersion. Exiting..."
212+
exit 1
213+
fi
214+
215+
if [ ! -f "$scheduler_config_file_path" ]; then
216+
echo "Critical error: $scheduler_config_file_path doesn't exist. Can't configure the scheduler for version $scheduler_image. Exiting..."
217+
exit 1
218+
fi
219+
# update the scheduler's version
220+
sed -i "s/XVERSIONX/$currentKubeSchedulerApiVersion/g" "$scheduler_config_file_path"
221+
222+
if [ ! -d "$scheduler_config_destination" ]; then
223+
echo "Critical error. $scheduler_config_destination doesn't exist. Please check the cluster configuration. Exiting..."
224+
exit 1
225+
fi
226+
227+
if ! $is_test ; then
228+
# copy the scheduler-config file to the expected folder
229+
echo "Will proceed to copy the scheduler configuration to its destination path: $scheduler_config_destination."
230+
cp "$scheduler_config_file_path" "$scheduler_config_destination"
231+
fi
232+
233+
# generate the new path of the config file
234+
scheduler_config_destination_path="$scheduler_config_destination/$scheduler_config_file"
235+
236+
## Add arguments to our kube-scheduler manifest. The arguments are:
237+
## 1) Config file with the extender policy and other configuration
238+
## 2) Mount the configuration file to make sure it's accessible by K8s
239+
sed -e "/ - kube-scheduler/a\\
240+
- --config=$scheduler_config_destination_path" "$MANIFEST_FILE" -i
241+
sed -e "/ volumeMounts:/a\\
242+
- mountPath: $scheduler_config_destination_path\n name: schedulerconfig\n readOnly: true" "$MANIFEST_FILE" -i
243+
sed -e "/ volumes:/a\\
244+
- hostPath:\n path: $scheduler_config_destination_path\n name: schedulerconfig" "$MANIFEST_FILE" -i
245+
fi
246+
247+
248+

‎telemetry-aware-scheduling/deploy/extender-configuration/scheduler-config.yaml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
apiVersion: kubescheduler.config.k8s.io/XVERSIONX
2+
kind: KubeSchedulerConfiguration
3+
clientConnection:
4+
kubeconfig: /etc/kubernetes/scheduler.conf
5+
extenders:
6+
- urlPrefix: "https://tas-service.default.svc.cluster.local:9001"
7+
prioritizeVerb: "scheduler/prioritize"
8+
filterVerb: "scheduler/filter"
9+
weight: 1
10+
enableHTTPS: true
11+
managedResources:
12+
- name: "telemetry/scheduling"
13+
ignoredByScheduler: true
14+
ignorable: true
15+
tlsConfig:
16+
insecure: false
17+
certFile: "/host/certs/client.crt"
18+
keyFile: "/host/certs/client.key"

0 commit comments

Comments
 (0)