28
28
envOverrides:
29
29
COMMON_VAR: role-value # overridden by role group below
30
30
ROLE_VAR: role-value # only defined here at role level
31
+ HDFS_NAMENODE_OPTS: >
32
+ -Djava.security.properties=/stackable/config/namenode/security.properties
33
+ -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8183:/stackable/jmx-exporter-config/namenode.yaml
34
+ -Xmx838860k
35
+ podOverrides:
36
+ spec:
37
+ initContainers:
38
+ - name: format-namenodes
39
+ volumeMounts:
40
+ - name: jmx-exporter-config
41
+ mountPath: /stackable/jmx-exporter-config
42
+ containers:
43
+ - name: namenode
44
+ volumeMounts:
45
+ - name: jmx-exporter-config
46
+ mountPath: /stackable/jmx-exporter-config
47
+ volumes:
48
+ - name: jmx-exporter-config
49
+ configMap:
50
+ name: jmx-exporter-config
31
51
config:
32
52
listenerClass: {{ test_scenario['values'] ['listener-class'] }}
33
53
logging:
42
62
envOverrides:
43
63
COMMON_VAR: role-value # overridden by role group below
44
64
ROLE_VAR: role-value # only defined here at role level
65
+ HDFS_DATANODE_OPTS: >
66
+ -Djava.security.properties=/stackable/config/datanode/security.properties
67
+ -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8082:/stackable/jmx-exporter-config/datanode.yaml
68
+ -Xmx419430k
69
+ podOverrides:
70
+ spec:
71
+ containers:
72
+ - name: datanode
73
+ volumeMounts:
74
+ - name: jmx-exporter-config
75
+ mountPath: /stackable/jmx-exporter-config
76
+ volumes:
77
+ - name: jmx-exporter-config
78
+ configMap:
79
+ name: jmx-exporter-config
45
80
config:
46
81
listenerClass: {{ test_scenario['values'] ['listener-class'] }}
47
82
logging:
@@ -71,6 +106,21 @@ spec:
71
106
envOverrides:
72
107
COMMON_VAR: role-value # overridden by role group below
73
108
ROLE_VAR: role-value # only defined here at role level
109
+ HDFS_JOURNALNODE_OPTS: >
110
+ -Djava.security.properties=/stackable/config/journalnode/security.properties
111
+ -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8081:/stackable/jmx-exporter-config/journalnode.yaml
112
+ -Xmx419430k
113
+ podOverrides:
114
+ spec:
115
+ containers:
116
+ - name: journalnode
117
+ volumeMounts:
118
+ - name: jmx-exporter-config
119
+ mountPath: /stackable/jmx-exporter-config
120
+ volumes:
121
+ - name: jmx-exporter-config
122
+ configMap:
123
+ name: jmx-exporter-config
74
124
config:
75
125
logging:
76
126
enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
@@ -93,3 +143,265 @@ spec:
93
143
# https://github.com/stackabletech/hdfs-operator/issues/514
94
144
- name: dashed-port
95
145
containerPort: 1234
146
+ ---
147
+ apiVersion: v1
148
+ kind: ConfigMap
149
+ metadata:
150
+ name: jmx-exporter-config
151
+ data:
152
+ namenode.yaml: |-
153
+ ---
154
+ startDelaySeconds: 10
155
+ ssl: false
156
+ lowercaseOutputName: true
157
+ lowercaseOutputLabelNames: true
158
+ whitelistObjectNames:
159
+ - 'Hadoop:service=NameNode,name=*'
160
+ - 'Hadoop:service=NameNode,name=MetricsSystem,sub=*'
161
+ blacklistObjectNames:
162
+ - 'Hadoop:service=NameNode,name=RetryCache.NameNodeRetryCache'
163
+ - 'Hadoop:service=NameNode,name=RpcActivity*'
164
+ - 'Hadoop:service=NameNode,name=RpcDetailedActivity*'
165
+ - 'Hadoop:service=NameNode,name=UgiMetrics'
166
+ rules:
167
+ # MetricsSystem
168
+ - pattern: 'Hadoop<service =(.*), name =MetricsSystem, sub =(.*) ><> (.*): (\d+)'
169
+ attrNameSnakeCase: true
170
+ name: hadoop_$1_$3
171
+ value: $4
172
+ labels:
173
+ service: HDFS
174
+ role: $1
175
+ kind: 'MetricsSystem'
176
+ sub: $2
177
+ type: GAUGE
178
+ # Total raw capacity in bytes, e.g. Hadoop:name=NameNodeInfo,attribute=Total
179
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (total): (\d+)'
180
+ attrNameSnakeCase: true
181
+ name: hadoop_$1_$3
182
+ value: $4
183
+ labels:
184
+ service: HDFS
185
+ role: $1
186
+ kind: $2
187
+ type: COUNTER
188
+ # Generic counter, e.g. Hadoop:name=FSNamesystem,attribute=FilesTotal
189
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_total): (\d+)'
190
+ attrNameSnakeCase: true
191
+ name: hadoop_$1_$3
192
+ value: $4
193
+ labels:
194
+ service: HDFS
195
+ role: $1
196
+ kind: $2
197
+ type: COUNTER
198
+ # Metrics suffixed with _created, e.g. Hadoop:name=NameNodeActivity,attribute=FilesCreated
199
+ # The suffix _created is reserved for timestamps, therefore an underscore is appended.
200
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_created): (.*)'
201
+ attrNameSnakeCase: true
202
+ name: hadoop_$1_$3_
203
+ value: $4
204
+ labels:
205
+ service: HDFS
206
+ role: $1
207
+ kind: $2
208
+ type: GAUGE
209
+ # Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
210
+ # The suffix _info is reserved for static information, therefore an underscore is appended.
211
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_info): (.*)'
212
+ attrNameSnakeCase: true
213
+ name: hadoop_$1_$3_
214
+ value: $4
215
+ labels:
216
+ service: HDFS
217
+ role: $1
218
+ kind: $2
219
+ type: GAUGE
220
+ # All other Hadoop metrics
221
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*): (.*)'
222
+ attrNameSnakeCase: true
223
+ name: hadoop_$1_$3
224
+ value: $4
225
+ labels:
226
+ service: HDFS
227
+ role: $1
228
+ kind: $2
229
+ type: GAUGE
230
+ datanode.yaml: |-
231
+ ---
232
+ startDelaySeconds: 10
233
+ ssl: false
234
+ lowercaseOutputName: true
235
+ lowercaseOutputLabelNames: true
236
+ whitelistObjectNames:
237
+ - 'Hadoop:service=DataNode,name=*'
238
+ - 'Hadoop:service=DataNode,name=MetricsSystem,sub=*'
239
+ blacklistObjectNames:
240
+ - 'Hadoop:service=DataNode,name=RpcActivity*'
241
+ - 'Hadoop:service=DataNode,name=RpcDetailedActivity*'
242
+ - 'Hadoop:service=DataNode,name=UgiMetrics'
243
+ rules:
244
+ # MetricsSystem
245
+ - pattern: 'Hadoop<service =(.*), name =MetricsSystem, sub =(.*) ><> (.*): (\d+)'
246
+ attrNameSnakeCase: true
247
+ name: hadoop_$1_$3
248
+ value: $4
249
+ labels:
250
+ service: HDFS
251
+ role: $1
252
+ kind: 'MetricsSystem'
253
+ sub: $2
254
+ type: GAUGE
255
+ # FSDatasetState (also extracts the FSDataset ID)
256
+ - pattern: 'Hadoop<service =(.*), name =FSDatasetState-(.*) ><> (.*): (\d+)'
257
+ attrNameSnakeCase: true
258
+ name: hadoop_$1_$3
259
+ value: $4
260
+ labels:
261
+ service: HDFS
262
+ role: $1
263
+ fsdatasetid: $2
264
+ kind: 'FSDatasetState'
265
+ type: GAUGE
266
+ # DataNodeActivity (also extracts hostname and port)
267
+ - pattern: 'Hadoop<service =(.*), name =DataNodeActivity-(.*)-(\d+) ><> (.*): (\d+)'
268
+ attrNameSnakeCase: true
269
+ name: hadoop_$1_$4
270
+ value: $5
271
+ labels:
272
+ service: HDFS
273
+ role: $1
274
+ host: $2
275
+ port: $3
276
+ kind: 'DataNodeActivity'
277
+ type: GAUGE
278
+ # Total raw capacity in bytes, e.g. Hadoop:name=NameNodeInfo,attribute=Total
279
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (total): (\d+)'
280
+ attrNameSnakeCase: true
281
+ name: hadoop_$1_$3
282
+ value: $4
283
+ labels:
284
+ service: HDFS
285
+ role: $1
286
+ kind: $2
287
+ type: COUNTER
288
+ # Generic counter, e.g. Hadoop:name=FSNamesystem,attribute=FilesTotal
289
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_total): (\d+)'
290
+ attrNameSnakeCase: true
291
+ name: hadoop_$1_$3
292
+ value: $4
293
+ labels:
294
+ service: HDFS
295
+ role: $1
296
+ kind: $2
297
+ type: COUNTER
298
+ # Metrics suffixed with _created, e.g. Hadoop:name=NameNodeActivity,attribute=FilesCreated
299
+ # The suffix _created is reserved for timestamps, therefore an underscore is appended.
300
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_created): (.*)'
301
+ attrNameSnakeCase: true
302
+ name: hadoop_$1_$3_
303
+ value: $4
304
+ labels:
305
+ service: HDFS
306
+ role: $1
307
+ kind: $2
308
+ type: GAUGE
309
+ # Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
310
+ # The suffix _info is reserved for static information, therefore an underscore is appended.
311
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_info): (.*)'
312
+ attrNameSnakeCase: true
313
+ name: hadoop_$1_$3_
314
+ value: $4
315
+ labels:
316
+ service: HDFS
317
+ role: $1
318
+ kind: $2
319
+ type: GAUGE
320
+ # All other Hadoop metrics
321
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*): (.*)'
322
+ attrNameSnakeCase: true
323
+ name: hadoop_$1_$3
324
+ value: $4
325
+ labels:
326
+ service: HDFS
327
+ role: $1
328
+ kind: $2
329
+ type: GAUGE
330
+ journalnode.yaml: |-
331
+ ---
332
+ startDelaySeconds: 10
333
+ ssl: false
334
+ lowercaseOutputName: true
335
+ lowercaseOutputLabelNames: true
336
+ whitelistObjectNames:
337
+ - 'Hadoop:service=JournalNode,name=*'
338
+ - 'Hadoop:service=JournalNode,name=MetricsSystem,sub=*'
339
+ blacklistObjectNames:
340
+ - 'Hadoop:service=JournalNode,name=RetryCache.JournalNodeRetryCache'
341
+ - 'Hadoop:service=JournalNode,name=RpcActivity*'
342
+ - 'Hadoop:service=JournalNode,name=RpcDetailedActivity*'
343
+ - 'Hadoop:service=JournalNode,name=UgiMetrics'
344
+ rules:
345
+ # MetricsSystem
346
+ - pattern: 'Hadoop<service =(.*), name =MetricsSystem, sub =(.*) ><> (.*): (\d+)'
347
+ attrNameSnakeCase: true
348
+ name: hadoop_$1_$3
349
+ value: $4
350
+ labels:
351
+ service: HDFS
352
+ role: $1
353
+ kind: 'MetricsSystem'
354
+ sub: $2
355
+ type: GAUGE
356
+ # Total raw capacity in bytes, e.g. Hadoop:name=NameNodeInfo,attribute=Total
357
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (total): (\d+)'
358
+ attrNameSnakeCase: true
359
+ name: hadoop_$1_$3
360
+ value: $4
361
+ labels:
362
+ service: HDFS
363
+ role: $1
364
+ kind: $2
365
+ type: COUNTER
366
+ # Generic counter, e.g. Hadoop:name=FSNamesystem,attribute=FilesTotal
367
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_total): (\d+)'
368
+ attrNameSnakeCase: true
369
+ name: hadoop_$1_$3
370
+ value: $4
371
+ labels:
372
+ service: HDFS
373
+ role: $1
374
+ kind: $2
375
+ type: COUNTER
376
+ # Metrics suffixed with _created, e.g. Hadoop:name=NameNodeActivity,attribute=FilesCreated
377
+ # The suffix _created is reserved for timestamps, therefore an underscore is appended.
378
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_created): (.*)'
379
+ attrNameSnakeCase: true
380
+ name: hadoop_$1_$3_
381
+ value: $4
382
+ labels:
383
+ service: HDFS
384
+ role: $1
385
+ kind: $2
386
+ type: GAUGE
387
+ # Metrics suffixed with _info, e.g. Hadoop:name=JvmMetrics,attribute=LogInfo
388
+ # The suffix _info is reserved for static information, therefore an underscore is appended.
389
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*_info): (.*)'
390
+ attrNameSnakeCase: true
391
+ name: hadoop_$1_$3_
392
+ value: $4
393
+ labels:
394
+ service: HDFS
395
+ role: $1
396
+ kind: $2
397
+ type: GAUGE
398
+ # All other Hadoop metrics
399
+ - pattern: 'Hadoop<service =(.*), name =(.*) ><> (.*): (.*)'
400
+ attrNameSnakeCase: true
401
+ name: hadoop_$1_$3
402
+ value: $4
403
+ labels:
404
+ service: HDFS
405
+ role: $1
406
+ kind: $2
407
+ type: GAUGE
0 commit comments