Skip to content

Commit e6a4516

Browse files
anjannathanjannath
committed
add systemd services for configuration after start
this adds 4 small systemd services that: - creates crc specific configurations for dnsmasq - sets a new uuid as cluster id - creates the pod for routes-controller - tries to grow the disk and filesystem - checks if the cluster operators are ready - adds the pull secret to the cluster - sets kubeadmin and developer user passwords - sets a custom ca for authentication - sets custom nip.io cluster domain
1 parent 914f90f commit e6a4516

22 files changed

+409
-0
lines changed

‎createdisk-library.sh

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,7 @@ function prepare_hyperV() {
216216
echo 'CONST{virt}=="microsoft", RUN{builtin}+="kmod load hv_sock"' > /etc/udev/rules.d/90-crc-vsock.rules
217217
EOF
218218
}
219+
219220
function prepare_qemu_guest_agent() {
220221
local vm_ip=$1
221222

@@ -400,3 +401,23 @@ function remove_pull_secret_from_disk() {
400401
esac
401402
}
402403

404+
function copy_systemd_units() {
405+
${SSH} core@${VM_IP} -- 'mkdir -p /home/core/systemd-units && mkdir -p /home/core/systemd-scripts'
406+
${SCP} systemd/crc-*.service core@${VM_IP}:/home/core/systemd-units/
407+
${SCP} systemd/crc-*.path core@${VM_IP}:/home/core/systemd-units/
408+
${SCP} systemd/crc-*.sh core@${VM_IP}:/home/core/systemd-scripts/
409+
410+
case "${BUNDLE_TYPE}" in
411+
"snc"|"okd")
412+
${SCP} systemd/ocp-*.service core@${VM_IP}:/home/core/systemd-units/
413+
${SCP} systemd/ocp-*.path core@${VM_IP}:/home/core/systemd-units/
414+
${SCP} systemd/ocp-*.sh core@${VM_IP}:/home/core/systemd-scripts/
415+
;;
416+
esac
417+
418+
${SSH} core@${VM_IP} -- 'sudo cp /home/core/systemd-units/* /etc/systemd/system/ && sudo cp /home/core/systemd-scripts/* /usr/local/bin/'
419+
${SSH} core@${VM_IP} -- 'ls /home/core/systemd-scripts/ | xargs -t -I % sudo chmod +x /usr/local/bin/%'
420+
${SSH} core@${VM_IP} -- 'sudo restorecon -rv /usr/local/bin'
421+
${SSH} core@${VM_IP} -- 'ls /home/core/systemd-units/ | xargs sudo systemctl enable'
422+
${SSH} core@${VM_IP} -- 'rm -rf /home/core/systemd-units /home/core/systemd-scripts'
423+
}

‎createdisk.sh

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,8 @@ if [ "${ARCH}" == "aarch64" ] && [ ${BUNDLE_TYPE} != "okd" ]; then
130130
${SSH} core@${VM_IP} -- "sudo rpm-ostree install https://kojipkgs.fedoraproject.org//packages/qemu/8.2.6/3.fc40/aarch64/qemu-user-static-x86-8.2.6-3.fc40.aarch64.rpm"
131131
fi
132132

133+
copy_systemd_units
134+
133135
cleanup_vm_image ${VM_NAME} ${VM_IP}
134136

135137
# Delete all the pods and lease from the etcd db so that when this bundle is use for the cluster provision, everything comes up in clean state.

‎systemd/crc-cluster-status.service

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=CRC Unit checking if cluster is ready
3+
After=kubelet.service
4+
Requires=kubelet.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/usr/local/bin/crc-cluster-status.sh
9+
RemainAfterExit=true
10+
11+
[Install]
12+
WantedBy=multi-user.target

‎systemd/crc-cluster-status.sh

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
export KUBECONFIG=/opt/kubeconfig
6+
7+
function check_cluster_unhealthy() {
8+
WAIT="authentication|console|etcd|ingress|openshift-apiserver"
9+
10+
until `oc get co > /dev/null 2>&1`
11+
do
12+
sleep 2
13+
done
14+
15+
for i in $(oc get co | grep -P "$WAIT" | awk '{ print $3 }')
16+
do
17+
if [[ $i == "False" ]]
18+
then
19+
return 0
20+
fi
21+
done
22+
return 1
23+
}
24+
25+
# rm -rf /tmp/.crc-cluster-ready
26+
27+
COUNTER=0
28+
CLUSTER_HEALTH_SLEEP=8
29+
CLUSTER_HEALTH_RETRIES=500
30+
31+
while $(check_cluster_unhealthy)
32+
do
33+
sleep $CLUSTER_HEALTH_SLEEP
34+
if [[ $COUNTER == $CLUSTER_HEALTH_RETRIES ]]
35+
then
36+
return 1
37+
fi
38+
((COUNTER++))
39+
done
40+
41+
# need to set a marker to let `crc` know the cluster is ready
42+
# touch /tmp/.crc-cluster-ready
43+

‎systemd/crc-dnsmasq.service

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
[Unit]
2+
Description=CRC Unit for configuring dnsmasq
3+
Requires=ovs-configuration.service
4+
After=ovs-configuration.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0"
9+
ExecStart=/usr/local/bin/crc-dnsmasq.sh
10+
ExecStartPost=/usr/bin/systemctl start dnsmasq.service
11+
12+
[Install]
13+
WantedBy=multi-user.target

‎systemd/crc-dnsmasq.sh

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
hostName=$(hostname)
6+
ip=$(ip -4 addr show br-ex | grep -oP '(?<=inet\s)192+(\.\d+){3}')
7+
iip=$(hostname -i)
8+
9+
cat << EOF > /etc/dnsmasq.d/crc-dnsmasq.conf
10+
listen-address=$ip
11+
expand-hosts
12+
log-queries
13+
local=/crc.testing/
14+
domain=crc.testing
15+
address=/apps-crc.testing/$ip
16+
address=/api.crc.testing/$ip
17+
address=/api-int.crc.testing/$ip
18+
address=/$hostName.crc.testing/$iip
19+
EOF
20+

‎systemd/crc-pullsecret.path

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[Unit]
2+
Description=CRC Unit for monitoring the pull secret path
3+
After=kubelet.service
4+
5+
[Path]
6+
PathExists=/opt/crc/pull-secret
7+
8+
[Install]
9+
WantedBy=multi-user.target

‎systemd/crc-pullsecret.service

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[Unit]
2+
Description=CRC Unit for adding pull secret to cluster
3+
After=kubelet.service
4+
Requires=kubelet.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/usr/local/bin/crc-pullsecret.sh
9+
10+
[Install]
11+
WantedBy=multi-user.target

‎systemd/crc-pullsecret.sh

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
export KUBECONFIG="/opt/kubeconfig"
6+
7+
retry=0
8+
max_retry=20
9+
until `oc get secret > /dev/null 2>&1`
10+
do
11+
[ $retry == $max_retry ] && exit 1
12+
sleep 5
13+
((retry++))
14+
done
15+
16+
# check if existing pull-secret is valid if not add the one from /opt/crc/pull-secret
17+
existingPsB64=$(oc get secret pull-secret -n openshift-config -o jsonpath="{['data']['\.dockerconfigjson']}")
18+
existingPs=$(echo "${existingPsB64}" | base64 -d)
19+
20+
echo "${existingPs}" | jq -e '.'
21+
22+
if [[ $? != 0 ]]; then
23+
pullSecretB64=$(cat /opt/crc/pull-secret)
24+
oc patch secret pull-secret -n openshift-config --type merge -p "{\"data\":{\".dockerconfigjson\":\"${pullSecretB64}\"}}"
25+
fi
26+

‎systemd/crc-routes-controller.service

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=CRC Unit starting routes controller
3+
After=kubelet.service
4+
Requires=kubelet.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecCondition=/usr/bin/bash -c "/usr/bin/ping -c1 gateway && exit 1 || exit 0"
9+
ExecStart=/usr/local/bin/crc-routes-controller.sh
10+
11+
[Install]
12+
WantedBy=multi-user.target

‎systemd/crc-routes-controller.sh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
export KUBECONFIG=/opt/kubeconfig
6+
7+
retry=0
8+
max_retry=20
9+
until `oc get pods > /dev/null 2>&1`
10+
do
11+
[ $retry == $max_retry ] && exit 1
12+
sleep 5
13+
((retry++))
14+
done
15+
16+
oc apply -f /opt/crc/routes-controller.yaml
17+

‎systemd/ocp-cluster-ca.service

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=CRC Unit setting custom cluster ca
3+
After=kubelet.service
4+
Requires=kubelet.service
5+
6+
[Service]
7+
Type=oneshot
8+
EnvironmentFile=/opt/crc/crc-cloud
9+
ExecStart=/usr/local/bin/ocp-cluster-ca.sh
10+
11+
[Install]
12+
WantedBy=multi-user.target

‎systemd/ocp-cluster-ca.sh

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
#!/bin/bash
2+
3+
# To generate the custom-ca.crt
4+
# USER="system:admin"
5+
# GROUP="system:masters"
6+
# USER_SUBJ="/O=${GROUP}/CN=${USER}"
7+
# NAME="custom"
8+
# CA_SUBJ="/OU=openshift/CN=admin-kubeconfig-signer-custom"
9+
# VALIDITY=3650
10+
11+
# openssl genrsa -out $NAME-ca.key 4096
12+
# openssl req -x509 -new -nodes -key $NAME-ca.key -sha256 -days $VALIDITY -out $NAME-ca.crt -subj "$CA_SUBJ"
13+
# openssl req -nodes -newkey rsa:2048 -keyout $USER.key -subj "$USER_SUBJ" -out $USER.csr
14+
# openssl x509 -extfile <(printf "extendedKeyUsage = clientAuth") -req -in $USER.csr \
15+
# -CA $NAME-ca.crt -CAkey $NAME-ca.key -CAcreateserial -out $USER.crt -days $VALIDITY -sha256
16+
17+
set -x
18+
19+
if [ -z $CRC_CLOUD ]; then
20+
exit 1
21+
fi
22+
23+
export KUBECONFIG="/opt/kubeconfig"
24+
25+
retry=0
26+
max_retry=20
27+
until `oc get configmap > /dev/null 2>&1`
28+
do
29+
[ $retry == $max_retry ] && exit 1
30+
sleep 5
31+
((retry++))
32+
done
33+
34+
custom_ca_path=/opt/crc/custom-ca.crt
35+
36+
retry=0
37+
max_retry=20
38+
until `ls ${custom_ca_path} > /dev/null 2>&1`
39+
do
40+
[ $retry == $max_retry ] && exit 1
41+
sleep 5
42+
((retry++))
43+
done
44+
45+
oc create configmap client-ca-custom -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path}
46+
oc patch apiserver cluster --type=merge -p '{"spec": {"clientCA": {"name": "client-ca-custom"}}}'
47+
oc create configmap admin-kubeconfig-client-ca -n openshift-config --from-file=ca-bundle.crt=${custom_ca_path} \
48+
--dry-run -o yaml | oc replace -f -
49+

‎systemd/ocp-clusterid.service

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
[Unit]
2+
Description=CRC Unit setting random cluster ID
3+
After=kubelet.service
4+
Requires=kubelet.service
5+
6+
[Service]
7+
Type=oneshot
8+
ExecStart=/usr/local/bin/ocp-clusterid.sh
9+
10+
[Install]
11+
WantedBy=multi-user.target

‎systemd/ocp-clusterid.sh

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
export KUBECONFIG="/opt/kubeconfig"
6+
uuid=$(uuidgen)
7+
8+
retry=0
9+
max_retry=20
10+
until `oc get clusterversion > /dev/null 2>&1`
11+
do
12+
[ $retry == $max_retry ] && exit 1
13+
sleep 5
14+
((retry++))
15+
done
16+
17+
oc patch clusterversion version -p "{\"spec\":{\"clusterID\":\"${uuid}\"}}" --type merge

‎systemd/ocp-custom-domain.service

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
[Unit]
2+
Description=CRC Unit setting nip.io domain for cluster
3+
After=kubelet.service
4+
Requires=kubelet.service
5+
6+
[Service]
7+
Type=oneshot
8+
EnvironmentFile=/opt/crc/crc-cloud
9+
ExecStart=/usr/local/bin/ocp-custom-domain.sh
10+
11+
[Install]
12+
WantedBy=multi-user.target

‎systemd/ocp-custom-domain.sh

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
if [ -z $CRC_CLOUD ]; then
6+
exit 1
7+
fi
8+
9+
export KUBECONFIG="/opt/kubeconfig"
10+
export EIP=$(hostname -i)
11+
12+
STEPS_SLEEP_TIME=30
13+
14+
retry=0
15+
max_retry=20
16+
until `oc get secret > /dev/null 2>&1`
17+
do
18+
[ $retry == $max_retry ] && exit 1
19+
sleep 5
20+
((retry++))
21+
done
22+
23+
# create cert and add as secret
24+
openssl req -newkey rsa:2048 -new -nodes -x509 -days 3650 -keyout nip.key -out nip.crt -subj "/CN=$EIP.nip.io" -addext "subjectAltName=DNS:apps.$EIP.nip.io,DNS:*.apps.$EIP.nip.io,DNS:api.$EIP.nip.io"
25+
oc create secret tls nip-secret --cert=nip.crt --key=nip.key -n openshift-config
26+
sleep $STEPS_SLEEP_TIME
27+
28+
# patch ingress
29+
cat <<EOF > ingress-patch.yaml
30+
spec:
31+
appsDomain: apps.$EIP.nip.io
32+
componentRoutes:
33+
- hostname: console-openshift-console.apps.$EIP.nip.io
34+
name: console
35+
namespace: openshift-console
36+
servingCertKeyPairSecret:
37+
name: nip-secret
38+
- hostname: oauth-openshift.apps.$EIP.nip.io
39+
name: oauth-openshift
40+
namespace: openshift-authentication
41+
servingCertKeyPairSecret:
42+
name: nip-secret
43+
EOF
44+
oc patch ingresses.config.openshift.io cluster --type=merge --patch-file=ingress-patch.yaml
45+
46+
# patch API server to use new CA secret
47+
oc patch apiserver cluster --type=merge -p '{"spec":{"servingCerts": {"namedCertificates":[{"names":["api.'$EIP'.nip.io"],"servingCertificate": {"name": "nip-secret"}}]}}}'
48+
49+
# patch image registry route
50+
oc patch -p '{"spec": {"host": "default-route-openshift-image-registry.'$EIP'.nip.io"}}' route default-route -n openshift-image-registry --type=merge
51+
52+
#wait_cluster_become_healthy "authentication|console|etcd|ingress|openshift-apiserver"

‎systemd/ocp-growfs.service

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
[Unit]
2+
Description=CRC Unit to grow the root filesystem
3+
4+
[Service]
5+
Type=oneshot
6+
ExecStart=/usr/local/bin/ocp-growfs.sh
7+
8+
[Install]
9+
WantedBy=multi-user.target

‎systemd/ocp-growfs.sh

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/bin/bash
2+
3+
set -x
4+
5+
root_partition=$(/usr/sbin/blkid -t TYPE=xfs -o device)
6+
/usr/bin/growpart "${root_partition#?}" "${root_partition#/dev/???}"
7+
8+
rootFS="/sysroot"
9+
mount -o remount,rw "${rootFS}"
10+
xfs_growfs "${rootFS}"
11+
#mount -o remount,ro "${rootFS}"

0 commit comments

Comments
 (0)