Skip to content

Commit 6a5712a

Browse files
bastjanbastjan
authored
Changes for vcluster dependency upgrades (#31)
* Use an emptyDir volume for some temporary k3s files * Update coredns integration from upstream helm chart * Simplify applying manifests to clusters with native methods
1 parent e2d96d5 commit 6a5712a

File tree

6 files changed

+581
-146
lines changed

6 files changed

+581
-146
lines changed

‎component/cluster.libsonnet

Lines changed: 43 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,24 @@ local cluster = function(name, options)
156156
},
157157
};
158158

159+
local initManifestsCM = kube.ConfigMap(name + '-init-manifests') {
160+
local manifests = options.additional_manifests,
161+
local manifestArray = if std.isArray(manifests) then
162+
manifests
163+
else if std.isObject(manifests) then
164+
std.objectValues(manifests)
165+
else
166+
error 'Manifests must be array or object'
167+
,
168+
169+
metadata+: {
170+
namespace: options.namespace,
171+
},
172+
data: {
173+
manifests: std.manifestYamlStream(manifestArray, false, false),
174+
},
175+
};
176+
159177
local statefulSet = kube.StatefulSet(name) {
160178
metadata+: {
161179
namespace: options.namespace,
@@ -199,12 +217,24 @@ local cluster = function(name, options)
199217
affinity: {},
200218
tolerations: [],
201219
serviceAccountName: 'vc-' + name,
202-
volumes: if !options.storage.persistence then [
220+
volumes: [
221+
{
222+
name: 'coredns',
223+
configMap: {
224+
name: 'vc-%s-coredns' % name,
225+
defaultMode: 420,
226+
},
227+
},
228+
{
229+
name: 'etc-rancher',
230+
emptyDir: {},
231+
},
232+
] + if !options.storage.persistence then [
203233
{
204234
name: 'data',
205235
emptyDir: {},
206236
},
207-
],
237+
] else [],
208238
local tlsSANs = [
209239
'--tls-san=%s.%s.svc.cluster.local' % [ name, options.namespace ],
210240
'--tls-san=%s.%s.svc' % [ name, options.namespace ],
@@ -248,6 +278,10 @@ local cluster = function(name, options)
248278
mountPath: '/data',
249279
name: 'data',
250280
},
281+
{
282+
mountPath: '/etc/rancher',
283+
name: 'etc-rancher',
284+
},
251285
],
252286
resources: {
253287
limits: {
@@ -295,6 +329,11 @@ local cluster = function(name, options)
295329
name: 'data',
296330
readOnly: true,
297331
},
332+
{
333+
mountPath: '/manifests/coredns',
334+
name: 'coredns',
335+
readOnly: true,
336+
},
298337
],
299338
resources: {
300339
limits: {
@@ -358,8 +397,9 @@ local cluster = function(name, options)
358397
service,
359398
headlessService,
360399
statefulSet,
400+
initManifestsCM,
401+
(import 'coredns.libsonnet').corednsConfigMap(name, options.namespace),
361402
if options.ingress.host != null then ingress,
362-
if std.length(options.additional_manifests) > 0 then postSetup.ApplyManifests(name, 'vc-%s-kubeconfig' % name, options.additional_manifests),
363403
if options.syn.registration_url != null then postSetup.Synthesize(name, 'vc-%s-kubeconfig' % name, options.syn.registration_url),
364404
] + if options.ocp_route.host != null then ocpRoute.RouteCreateJob(name, 'vc-%s-kubeconfig' % name, options.ocp_route.host) else []);
365405

‎component/coredns.libsonnet

Lines changed: 238 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,238 @@
1+
local kap = import 'lib/kapitan.libjsonnet';
2+
local kube = import 'lib/kube.libjsonnet';
3+
local inv = kap.inventory();
4+
local common = import 'common.libsonnet';
5+
// The hiera parameters for the component
6+
local params = inv.parameters.vcluster;
7+
8+
local corednsConfigMap =
9+
function(name, namespace)
10+
kube.ConfigMap('vc-%s-coredns' % name) {
11+
metadata+: {
12+
namespace: namespace,
13+
},
14+
data: {
15+
// The deployment has some variables in there that get modified by vcluster.
16+
// It is not valid yaml, so we need to use a string.
17+
// The Helm chart does use a string too.
18+
'coredns.yaml': |||
19+
apiVersion: v1
20+
kind: ServiceAccount
21+
metadata:
22+
name: coredns
23+
namespace: kube-system
24+
---
25+
apiVersion: rbac.authorization.k8s.io/v1
26+
kind: ClusterRole
27+
metadata:
28+
labels:
29+
kubernetes.io/bootstrapping: rbac-defaults
30+
name: system:coredns
31+
rules:
32+
- apiGroups:
33+
- ""
34+
resources:
35+
- endpoints
36+
- services
37+
- pods
38+
- namespaces
39+
verbs:
40+
- list
41+
- watch
42+
- apiGroups:
43+
- discovery.k8s.io
44+
resources:
45+
- endpointslices
46+
verbs:
47+
- list
48+
- watch
49+
---
50+
apiVersion: rbac.authorization.k8s.io/v1
51+
kind: ClusterRoleBinding
52+
metadata:
53+
annotations:
54+
rbac.authorization.kubernetes.io/autoupdate: "true"
55+
labels:
56+
kubernetes.io/bootstrapping: rbac-defaults
57+
name: system:coredns
58+
roleRef:
59+
apiGroup: rbac.authorization.k8s.io
60+
kind: ClusterRole
61+
name: system:coredns
62+
subjects:
63+
- kind: ServiceAccount
64+
name: coredns
65+
namespace: kube-system
66+
---
67+
apiVersion: v1
68+
kind: ConfigMap
69+
metadata:
70+
name: coredns
71+
namespace: kube-system
72+
data:
73+
Corefile: |
74+
.:1053 {
75+
{{.LOG_IN_DEBUG}}
76+
errors
77+
health
78+
ready
79+
kubernetes cluster.local in-addr.arpa ip6.arpa {
80+
pods insecure
81+
fallthrough in-addr.arpa ip6.arpa
82+
}
83+
hosts /etc/coredns/NodeHosts {
84+
ttl 60
85+
reload 15s
86+
fallthrough
87+
}
88+
prometheus :9153
89+
forward . /etc/resolv.conf
90+
cache 30
91+
loop
92+
reload
93+
loadbalance
94+
}
95+
96+
import /etc/coredns/custom/*.server
97+
NodeHosts: ""
98+
---
99+
apiVersion: apps/v1
100+
kind: Deployment
101+
metadata:
102+
name: coredns
103+
namespace: kube-system
104+
labels:
105+
k8s-app: kube-dns
106+
kubernetes.io/name: "CoreDNS"
107+
spec:
108+
replicas: 1
109+
strategy:
110+
type: RollingUpdate
111+
rollingUpdate:
112+
maxUnavailable: 1
113+
selector:
114+
matchLabels:
115+
k8s-app: kube-dns
116+
template:
117+
metadata:
118+
labels:
119+
k8s-app: kube-dns
120+
spec:
121+
priorityClassName: "system-cluster-critical"
122+
serviceAccountName: coredns
123+
nodeSelector:
124+
kubernetes.io/os: linux
125+
topologySpreadConstraints:
126+
- maxSkew: 1
127+
topologyKey: kubernetes.io/hostname
128+
whenUnsatisfiable: DoNotSchedule
129+
labelSelector:
130+
matchLabels:
131+
k8s-app: kube-dns
132+
containers:
133+
- name: coredns
134+
image: {{.IMAGE}}
135+
imagePullPolicy: IfNotPresent
136+
resources:
137+
limits:
138+
cpu: 1000m
139+
memory: 170Mi
140+
requests:
141+
cpu: 100m
142+
memory: 70Mi
143+
args: [ "-conf", "/etc/coredns/Corefile" ]
144+
volumeMounts:
145+
- name: config-volume
146+
mountPath: /etc/coredns
147+
readOnly: true
148+
- name: custom-config-volume
149+
mountPath: /etc/coredns/custom
150+
readOnly: true
151+
ports:
152+
- containerPort: 1053
153+
name: dns
154+
protocol: UDP
155+
- containerPort: 1053
156+
name: dns-tcp
157+
protocol: TCP
158+
- containerPort: 9153
159+
name: metrics
160+
protocol: TCP
161+
securityContext:
162+
runAsUser: {{.RUN_AS_USER}}
163+
runAsNonRoot: {{.RUN_AS_NON_ROOT}}
164+
allowPrivilegeEscalation: false
165+
capabilities:
166+
drop:
167+
- ALL
168+
readOnlyRootFilesystem: true
169+
livenessProbe:
170+
httpGet:
171+
path: /health
172+
port: 8080
173+
scheme: HTTP
174+
initialDelaySeconds: 60
175+
periodSeconds: 10
176+
timeoutSeconds: 1
177+
successThreshold: 1
178+
failureThreshold: 3
179+
readinessProbe:
180+
httpGet:
181+
path: /ready
182+
port: 8181
183+
scheme: HTTP
184+
initialDelaySeconds: 0
185+
periodSeconds: 2
186+
timeoutSeconds: 1
187+
successThreshold: 1
188+
failureThreshold: 3
189+
dnsPolicy: Default
190+
volumes:
191+
- name: config-volume
192+
configMap:
193+
name: coredns
194+
items:
195+
- key: Corefile
196+
path: Corefile
197+
- key: NodeHosts
198+
path: NodeHosts
199+
- name: custom-config-volume
200+
configMap:
201+
name: coredns-custom
202+
optional: true
203+
---
204+
apiVersion: v1
205+
kind: Service
206+
metadata:
207+
name: kube-dns
208+
namespace: kube-system
209+
annotations:
210+
prometheus.io/port: "9153"
211+
prometheus.io/scrape: "true"
212+
labels:
213+
k8s-app: kube-dns
214+
kubernetes.io/cluster-service: "true"
215+
kubernetes.io/name: "CoreDNS"
216+
spec:
217+
selector:
218+
k8s-app: kube-dns
219+
type: ClusterIP
220+
ports:
221+
- name: dns
222+
port: 53
223+
targetPort: 1053
224+
protocol: UDP
225+
- name: dns-tcp
226+
port: 53
227+
targetPort: 1053
228+
protocol: TCP
229+
- name: metrics
230+
port: 9153
231+
protocol: TCP
232+
|||,
233+
},
234+
};
235+
236+
{
237+
corednsConfigMap: corednsConfigMap,
238+
}

‎component/post-setup.libsonnet

Lines changed: 0 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -42,52 +42,6 @@ local synthesize = function(name, secretName, url)
4242
},
4343
};
4444

45-
local applyManifests = function(name, secretName, manifests)
46-
local jobName = '%s-apply-manifests' % name;
47-
local manifestArray = if std.isArray(manifests) then
48-
manifests
49-
else if std.isObject(manifests) then
50-
std.objectValues(manifests)
51-
else
52-
error 'Manifests must be array or object'
53-
;
54-
kube.Job(jobName) {
55-
metadata+: {
56-
namespace: params.namespace,
57-
annotations+: {
58-
'argocd.argoproj.io/hook': 'PostSync',
59-
},
60-
},
61-
spec+: {
62-
template+: {
63-
spec+: {
64-
containers_+: {
65-
patch_crds: kube.Container(jobName) {
66-
image: common.formatImage(params.images.kubectl),
67-
workingDir: '/export',
68-
command: [ 'sh' ],
69-
args: [ '-eu', '-c', importstr './scripts/apply.sh', '--' ] + std.map(function(m) std.manifestJsonEx(m, ''), manifestArray),
70-
env: [
71-
{ name: 'HOME', value: '/export' },
72-
{ name: 'VCLUSTER_SERVER_URL', value: 'https://%s:443' % name },
73-
],
74-
volumeMounts: [
75-
{ name: 'export', mountPath: '/export' },
76-
{ name: 'kubeconfig', mountPath: '/etc/vcluster-kubeconfig', readOnly: true },
77-
],
78-
},
79-
},
80-
volumes+: [
81-
{ name: 'export', emptyDir: {} },
82-
{ name: 'kubeconfig', secret: { secretName: secretName } },
83-
],
84-
},
85-
},
86-
},
87-
};
88-
89-
9045
{
9146
Synthesize: synthesize,
92-
ApplyManifests: applyManifests,
9347
}

0 commit comments

Comments
 (0)