Skip to content

Commit 6bbb676

Browse files
DebakelOrakelDebakelOrakel
authored
Merge pull request #1 from projectsyn/initial-implementation
Initial implementation
2 parents 8f2bfc3 + 14a2fa9 commit 6bbb676

File tree

20 files changed

+625
-5
lines changed

20 files changed

+625
-5
lines changed

‎.cruft.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@
77
"name": "cloudscale-cloud-controller-manager",
88
"slug": "cloudscale-cloud-controller-manager",
99
"parameter_key": "cloudscale_cloud_controller_manager",
10-
"test_cases": "defaults",
10+
"test_cases": "defaults openshift4",
1111
"add_lib": "n",
1212
"add_pp": "n",
1313
"add_golden": "y",

‎.github/workflows/test.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ jobs:
3333
matrix:
3434
instance:
3535
- defaults
36+
- openshift4
3637
defaults:
3738
run:
3839
working-directory: ${{ env.COMPONENT_NAME }}
@@ -48,6 +49,7 @@ jobs:
4849
matrix:
4950
instance:
5051
- defaults
52+
- openshift4
5153
defaults:
5254
run:
5355
working-directory: ${{ env.COMPONENT_NAME }}

‎Makefile.vars.mk

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,4 +50,4 @@ KUBENT_IMAGE ?= ghcr.io/doitintl/kube-no-trouble:latest
5050
KUBENT_DOCKER ?= $(DOCKER_CMD) $(DOCKER_ARGS) $(root_volume) --entrypoint=/app/kubent $(KUBENT_IMAGE)
5151

5252
instance ?= defaults
53-
test_instances = tests/defaults.yml
53+
test_instances = tests/defaults.yml tests/openshift4.yml

‎class/cloudscale-cloud-controller-manager.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
parameters:
22
kapitan:
3+
dependencies:
4+
- type: https
5+
source: https://github.com/cloudscale-ch/cloudscale-cloud-controller-manager/releases/download/${cloudscale_cloud_controller_manager:manifests_version}/config.yml
6+
output_path: ${_base_directory}/manifests/${cloudscale_cloud_controller_manager:manifests_version}/config.yml
7+
38
compile:
49
- input_paths:
510
- ${_base_directory}/component/app.jsonnet

‎class/defaults.yml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,7 @@ parameters:
22
cloudscale_cloud_controller_manager:
33
=_metadata: {}
44
namespace: syn-cloudscale-cloud-controller-manager
5+
6+
manifests_version: 1.0.0
7+
8+
api_token: ?{vaultkv:${cluster:tenant}/${cluster:name}/cloudscale/token}

‎component/main.jsonnet

Lines changed: 100 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,107 @@
1-
// main template for cloudscale-cloud-controller-manager
21
local kap = import 'lib/kapitan.libjsonnet';
32
local kube = import 'lib/kube.libjsonnet';
43
local inv = kap.inventory();
5-
// The hiera parameters for the component
4+
65
local params = inv.parameters.cloudscale_cloud_controller_manager;
6+
local isOpenShift = inv.parameters.facts.distribution == 'openshift4';
7+
8+
local manifests = std.parseJson(
9+
kap.yaml_load_stream(
10+
'%s/manifests/%s/config.yml'
11+
% [ inv.parameters._base_directory, params.manifests_version ]
12+
)
13+
);
14+
15+
local patchDaemonset(obj) =
16+
if isOpenShift && obj.kind == 'DaemonSet' then
17+
obj {
18+
spec+: {
19+
template+: {
20+
spec+: {
21+
nodeSelector: {
22+
'node-role.kubernetes.io/master': '',
23+
},
24+
tolerations+: [
25+
{
26+
key: 'node-role.kubernetes.io/master',
27+
effect: 'NoSchedule',
28+
},
29+
],
30+
},
31+
},
32+
},
33+
}
34+
else
35+
obj;
36+
37+
local tokenSecret = kube.Secret('cloudscale') {
38+
metadata+: {
39+
namespace: params.namespace,
40+
},
41+
data:: {},
42+
stringData: {
43+
'access-token': params.api_token,
44+
},
45+
};
46+
47+
local customRBAC = if isOpenShift then
48+
[
49+
kube.RoleBinding('ccm-hostnetwork') {
50+
metadata+: {
51+
// Required if we want to deploy this manifest during cluster
52+
// bootstrap.
53+
namespace: params.namespace,
54+
},
55+
roleRef_: kube.ClusterRole('system:openshift:scc:hostnetwork'),
56+
subjects: [
57+
{
58+
kind: 'ServiceAccount',
59+
name: std.filter(
60+
function(obj) obj.kind == 'DaemonSet', manifests
61+
)[0].spec.template.spec.serviceAccountName,
62+
namespace: params.namespace,
63+
},
64+
],
65+
},
66+
]
67+
else
68+
[];
769

8-
// Define outputs below
970
{
71+
[if params.namespace != 'kube-system' then '00_namespace']:
72+
kube.Namespace(params.namespace) {
73+
metadata+: {
74+
annotations+: {
75+
// NOTE(sg): we set this unconditionally since it doesn't matter on
76+
// non-OCP.
77+
'openshift.io/node-selector': '',
78+
},
79+
},
80+
},
81+
'01_secret': tokenSecret,
82+
'10_daemonset': [
83+
patchDaemonset(object) {
84+
metadata+: {
85+
namespace: params.namespace,
86+
},
87+
}
88+
for object in manifests
89+
if std.setMember(object.kind, [ 'DaemonSet', 'ServiceAccount' ])
90+
],
91+
'20_rbac': [
92+
object + if std.objectHas(object, 'subjects') then
93+
{
94+
subjects: [
95+
sub {
96+
namespace: params.namespace,
97+
}
98+
for sub in super.subjects
99+
],
100+
}
101+
else
102+
{}
103+
for object in manifests
104+
if std.setMember(object.kind, [ 'ClusterRole', 'ClusterRoleBinding' ])
105+
],
106+
[if std.length(customRBAC) > 0 then '30_custom_rbac']: customRBAC,
10107
}

‎docs/modules/ROOT/pages/how-tos/deploy-ocp.adoc

Lines changed: 138 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
= Deploy on existing OpenShift cluster
2+
3+
This guide describes how to deploy cloudscale.ch CCM on an existing cluster.
4+
5+
== Steb-by-step guide
6+
7+
. Ensure nodes are registered with cloudscale.ch provider id
8+
+
9+
If not already present add the following machine config which customizes the kubelet systemd service on all cluster nodes.
10+
As described in https://github.com/openshift/infrastructure-provider-onboarding-guide/blob/main/docs/platform-external/installing.md.
11+
+
12+
[source,yaml]
13+
----
14+
spec:
15+
config:
16+
ignition:
17+
version: 3.2.0
18+
storage:
19+
files:
20+
- mode: 493 # 0755
21+
path: /usr/local/bin/appuio-kubelet-providerid
22+
contents:
23+
inline: |
24+
#!/bin/bash
25+
set -e -o pipefail
26+
NODECONF=/etc/systemd/system/kubelet.service.d/20-appuio-providerid.conf
27+
if [ -e "\${NODECONF}" ]; then
28+
echo "Not replacing existing \${NODECONF}"
29+
exit 0
30+
fi
31+
PROVIDERID=$(curl -sL http://169.254.169.254/openstack/2017-02-22/meta_data.json | \
32+
jq -r .meta.cloudscale_uuid)
33+
cat >"\${NODECONF}" <<EOF
34+
[Service]
35+
Environment="KUBELET_PROVIDERID=cloudscale://\${PROVIDERID}"
36+
EOF
37+
systemd:
38+
units:
39+
- name: appuio-kubelet-providerid.service
40+
enabled: true
41+
contents: |
42+
[Unit]
43+
Description=Fetch provide id from metadata service
44+
After=NetworkManager-wait-online.service
45+
Before=kubelet.service
46+
[Service]
47+
ExecStart=/usr/local/bin/appuio-kubelet-providerid
48+
Type=oneshot
49+
[Install]
50+
WantedBy=network-online.target
51+
----
52+
53+
. Check if provider id is present on nodes
54+
+
55+
[source,bash]
56+
----
57+
for n in $(kubectl get nodes -oname); do
58+
PROVIDERID=$(oc -n syn-debug-nodes --as=cluster-admin debug $n -- chroot /host cat /etc/systemd/system/kubelet.service.d/20-appuio-providerid.conf 2>&1 | grep PROVIDERID | sed -e 's/^Environment="KUBELET_PROVIDERID=\([^"]\+\)"$/\1/g')
59+
echo kubectl --as=cluster-admin patch $n --type=merge -p "{\"spec\":{\"providerID\":\"${PROVIDERID}\"}}";
60+
done
61+
----
62+
63+
. Patch nodes with provider id
64+
+
65+
[source,bash]
66+
----
67+
for n in $(kubectl get nodes -oname); do
68+
PROVIDERID=$(oc -n syn-debug-nodes --as=cluster-admin debug $n -- chroot /host cat /etc/systemd/system/kubelet.service.d/20-appuio-providerid.conf 2>&1 | grep PROVIDERID | sed -e 's/^Environment="KUBELET_PROVIDERID=\([^"]\+\)"$/\1/g')
69+
kubectl --as=cluster-admin patch $n --type=merge -p "{\"spec\":{\"providerID\":\"${PROVIDERID}\"}}";
70+
done
71+
kubectl get no -ocustom-columns='NAME:.metadata.name,PROVIDER_ID:.spec.providerID'
72+
----
73+
+
74+
[source,bash]
75+
----
76+
NAME PROVIDER_ID
77+
infra-8344 cloudscale://1b04b1fb-d6c8-4108-92f4-f64f8c1ca960
78+
infra-87c9 cloudscale://2813282a-2a88-461d-a927-4ad96bf4360a
79+
infra-eba2 cloudscale://22fd8f7d-0eb8-4d10-82d7-e35df65e62b8
80+
master-3b79 cloudscale://b7597a27-0129-40cc-9bd8-ac19f5d3deb6
81+
master-6cc2 cloudscale://556ca8bd-37a3-4a48-8412-8eafe90c606a
82+
master-edc2 cloudscale://48255c88-ad2b-4242-a4ba-74332f16b264
83+
storage-17fd cloudscale://2584f7f1-a2cf-4598-903b-80ff6acc1dd0
84+
storage-534a cloudscale://e2c96748-7d24-45e7-b345-8b83d7dc21bf
85+
storage-c4e4 cloudscale://a8358646-fe30-4fb6-bffb-6971863286d3
86+
worker-10db cloudscale://f2a2cc9b-aae9-4674-ae64-de4bd3cfb1db
87+
worker-3b6c cloudscale://671553ab-2243-4075-b35b-0592aa9d2299
88+
worker-52c6 cloudscale://a6255254-46bf-4c4f-8638-9eb26559814e
89+
worker-8e76 cloudscale://a0ba2200-bfc3-435a-b733-8607b6ee4190
90+
----
91+
92+
. Deploy component-cloudscale-cloud-controller-manager
93+
94+
. Patch infrastructure config
95+
+
96+
[source,bash]
97+
----
98+
kubectl --as cluster-admin patch infrastructure.config cluster --type=merge -p '{"spec":{"platformSpec":{"external":{"platformName":"cloudscale.ch"},"type":"External"}}}'
99+
infrastructure.config.openshift.io/cluster patched
100+
----
101+
+
102+
[source,bash]
103+
----
104+
kubectl proxy &
105+
curl -XPATCH -H"Content-Type: application/merge-patch+json" http://localhost:8001/apis/config.openshift.io/v1/infrastructures/cluster/status -d '{"status":{"platform":"External","platformStatus":{"external":{"cloudControllerManager":{"state":"External"}},"type":"External"}}}'
106+
----
107+
108+
. Taint nodes with `node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule`
109+
+
110+
[source,bash]
111+
----
112+
kubectl --as cluster-admin taint node --all node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule
113+
----
114+
115+
. Check if instance-type is applied
116+
+
117+
[source,bash]
118+
----
119+
kubectl get nodes -ocustom-columns='NAME:.metadata.name,INSTANCE_TYPE:.metadata.labels.node\.kubernetes\.io/instance-type'
120+
----
121+
+
122+
[source,bash]
123+
----
124+
NAME INSTANCE_TYPE
125+
infra-8344 flex-24-6
126+
infra-87c9 flex-24-6
127+
infra-eba2 flex-24-6
128+
master-3b79 flex-24-6
129+
master-6cc2 flex-24-6
130+
master-edc2 flex-24-6
131+
storage-17fd flex-8-2
132+
storage-534a flex-8-2
133+
storage-c4e4 flex-8-2
134+
worker-10db flex-16-4
135+
worker-3b6c flex-16-4
136+
worker-52c6 flex-16-4
137+
worker-8e76 flex-16-4
138+
----

‎docs/modules/ROOT/pages/references/parameters.adoc

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,16 @@ default:: `syn-cloudscale-cloud-controller-manager`
1111
The namespace in which to deploy this component.
1212

1313

14+
== `api_token`
15+
16+
[horizontal]
17+
type:: string
18+
default:: `?{vaultkv:${cluster:tenant}/${cluster:name}/cloudscale/token}`
19+
20+
cloudscale.ch API token to be used by the CCM driver.
21+
This should be a reference to a secret in Vault instead of the plaintext token.
22+
23+
1424
== Example
1525

1626
[source,yaml]

‎docs/modules/ROOT/partials/nav.adoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11
* xref:index.adoc[Home]
2+
* xref:how-tos/deploy-ocp.adoc[]
23
* xref:references/parameters.adoc[Parameters]

‎tests/golden/defaults/cloudscale-cloud-controller-manager/cloudscale-cloud-controller-manager/00_namespace.yaml

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
apiVersion: v1
2+
kind: Namespace
3+
metadata:
4+
annotations:
5+
openshift.io/node-selector: ''
6+
labels:
7+
name: syn-cloudscale-cloud-controller-manager
8+
name: syn-cloudscale-cloud-controller-manager

0 commit comments

Comments
 (0)