test: Check metrics in smoke test

Author: siegfriedweber

tests/templates/kuttl/smoke/30-install-hdfs.yaml.j2

@@ -252,6 +252,18 @@ data:
           kind: 'MetricsSystem'
           sub: $2
         type: GAUGE
+      # FSDatasetState with _total suffix (also extracts the FSDataset ID),
+      # e.g. Hadoop:name=FSDatasetState,attribute=EstimatedCapacityLostTotal
+      - pattern: 'Hadoop<service=(.*), name=FSDatasetState-(.*)><>(.*_total): (\d+)'
+        attrNameSnakeCase: true
+        name: hadoop_$1_$3
+        value: $4
+        labels:
+          service: HDFS
+          role: $1
+          fsdatasetid: $2
+          kind: 'FSDatasetState'
+        type: COUNTER
       # FSDatasetState (also extracts the FSDataset ID)
       - pattern: 'Hadoop<service=(.*), name=FSDatasetState-(.*)><>(.*): (\d+)'
         attrNameSnakeCase: true
@@ -263,7 +275,19 @@ data:
           fsdatasetid: $2
           kind: 'FSDatasetState'
         type: GAUGE
-      # DataNodeActivity (also extracts hostname and port)
+      # DataNodeActivity with _info suffix (also extracts hostname and port),
+      # e.g. Hadoop:name=DataNodeActivity-hdfs-datanode-default-0-9866,attribute=BlocksGetLocalPathInfo
+      - pattern: 'Hadoop<service=(.*), name=DataNodeActivity-(.*)-(\d+)><>(.*_info): (\d+)'
+        attrNameSnakeCase: true
+        name: hadoop_$1_$4_
+        value: $5
+        labels:
+          service: HDFS
+          role: $1
+          host: $2
+          port: $3
+          kind: 'DataNodeActivity'
+        type: GAUGE
       - pattern: 'Hadoop<service=(.*), name=DataNodeActivity-(.*)-(\d+)><>(.*): (\d+)'
         attrNameSnakeCase: true
         name: hadoop_$1_$4

tests/templates/kuttl/smoke/40-assert.yaml

@@ -6,7 +6,7 @@ timeout: 300
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: webhdfs
+  name: test-runner
 status:
   readyReplicas: 1
   replicas: 1

tests/templates/kuttl/smoke/40-webhdfs.yaml renamed to tests/templates/kuttl/smoke/40-install-test-runner.yaml

@@ -2,21 +2,21 @@
 apiVersion: apps/v1
 kind: StatefulSet
 metadata:
-  name: webhdfs
+  name: test-runner
   labels:
-    app: webhdfs
+    app: test-runner
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app: webhdfs
+      app: test-runner
   template:
     metadata:
       labels:
-        app: webhdfs
+        app: test-runner
     spec:
       containers:
-        - name: webhdfs
+        - name: test-runner
           image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
           stdin: true
           tty: true

tests/templates/kuttl/smoke/50-assert.yaml

@@ -2,4 +2,4 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestAssert
 commands:
-  - script: kubectl exec -n $NAMESPACE webhdfs-0 -- python /tmp/webhdfs.py $NAMESPACE ls
+  - script: kubectl exec -n $NAMESPACE test-runner-0 -- python /tmp/webhdfs.py $NAMESPACE ls

tests/templates/kuttl/smoke/50-create-file.yaml

@@ -2,6 +2,6 @@
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  - script: kubectl cp -n $NAMESPACE ./webhdfs.py  webhdfs-0:/tmp
-  - script: kubectl cp -n $NAMESPACE ./testdata.txt webhdfs-0:/tmp
-  - script: kubectl exec -n $NAMESPACE webhdfs-0 -- python /tmp/webhdfs.py $NAMESPACE create
+  - script: kubectl cp -n $NAMESPACE ./webhdfs.py  test-runner-0:/tmp
+  - script: kubectl cp -n $NAMESPACE ./testdata.txt test-runner-0:/tmp
+  - script: kubectl exec -n $NAMESPACE test-runner-0 -- python /tmp/webhdfs.py $NAMESPACE create

tests/templates/kuttl/smoke/51-assert.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestAssert
+commands:
+  - script: kubectl exec -n $NAMESPACE test-runner-0 -- python /tmp/metrics-test.py $NAMESPACE

tests/templates/kuttl/smoke/51-copy-metrics-test-script.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: kubectl cp -n $NAMESPACE ./metrics-test.py test-runner-0:/tmp

tests/templates/kuttl/smoke/metrics-test.py

@@ -0,0 +1,96 @@
+from requests import Response
+import re
+import requests
+import sys
+import logging
+
+
+def check_metrics(
+        namespace: str,
+        role: str,
+        port: int,
+        expected_metrics: list[str]
+    ) -> None:
+    response: Response = requests.get(
+        f'http://hdfs-{role}-default-0.hdfs-{role}-default.{namespace}.svc.cluster.local:{port}/metrics'
+    )
+    assert response.ok, "Requesting metrics failed"
+
+    for metric in expected_metrics:
+        assert re.search(f'^{metric}', response.text, re.MULTILINE) is not None, \
+                f"Metric '{metric}' not found for {role}"
+
+
+def check_namenode_metrics(
+        namespace: str,
+    ) -> None:
+    expected_metrics = [
+      # Kind "MetricsSystem"
+      'hadoop_namenode_num_active_sources{kind="MetricsSystem",role="NameNode",service="HDFS",sub="Stats"}',
+      # Attribute "Total"
+      'hadoop_namenode_total{kind="NameNodeInfo",role="NameNode",service="HDFS"}',
+      # Counter suffixed with "_total"
+      'hadoop_namenode_files_total{kind="FSNamesystem",role="NameNode",service="HDFS"}',
+      # Metric suffixed with "_created"
+      'hadoop_namenode_files_created_{kind="NameNodeActivity",role="NameNode",service="HDFS"}',
+      # Metric suffixed with "_info"
+      'hadoop_namenode_log_info_{kind="JvmMetrics",role="NameNode",service="HDFS"}',
+      # Non-special metric
+      'hadoop_namenode_files_deleted{kind="NameNodeActivity",role="NameNode",service="HDFS"}',
+    ]
+    check_metrics(namespace, 'namenode', 8183, expected_metrics)
+
+
+def check_datanode_metrics(
+        namespace: str,
+    ) -> None:
+    expected_metrics = [
+      # Kind "MetricsSystem"
+      'hadoop_datanode_num_active_sources{kind="MetricsSystem",role="DataNode",service="HDFS",sub="Stats"}',
+      # Kind "FSDatasetState" suffixed with "_total"
+      'hadoop_datanode_estimated_capacity_lost_total{fsdatasetid=".+",kind="FSDatasetState",role="DataNode",service="HDFS"}',
+      # Kind "FSDatasetState"
+      'hadoop_datanode_capacity{fsdatasetid=".+",kind="FSDatasetState",role="DataNode",service="HDFS"}',
+      # Kind "DataNodeActivity" suffixed with "_info"
+      'hadoop_datanode_blocks_get_local_path_info_{host="hdfs-datanode-default-0\\.hdfs-datanode-default\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
+      # Kind "DataNodeActivity"
+      'hadoop_datanode_blocks_read{host="hdfs-datanode-default-0\\.hdfs-datanode-default\\..+\\.svc\\.cluster\\.local",kind="DataNodeActivity",port="9866",role="DataNode",service="HDFS"}',
+      # Counter suffixed with "_total"
+      'hadoop_datanode_estimated_capacity_lost_total{kind="FSDatasetState",role="DataNode",service="HDFS"}',
+      # Metric suffixed with "_info"
+      'hadoop_datanode_log_info_{kind="JvmMetrics",role="DataNode",service="HDFS"}',
+      # Non-special metric
+      'hadoop_datanode_gc_count{kind="JvmMetrics",role="DataNode",service="HDFS"}',
+    ]
+    check_metrics(namespace, 'datanode', 8082, expected_metrics)
+
+
+def check_journalnode_metrics(
+        namespace: str,
+    ) -> None:
+    expected_metrics = [
+      # Kind "MetricsSystem"
+      'hadoop_journalnode_num_active_sources{kind="MetricsSystem",role="JournalNode",service="HDFS",sub="Stats"}',
+      # Metric suffixed with "_info"
+      'hadoop_journalnode_log_info_{kind="JvmMetrics",role="JournalNode",service="HDFS"}',
+      # Non-special metric
+      'hadoop_journalnode_bytes_written{kind="Journal-hdfs",role="JournalNode",service="HDFS"}',
+    ]
+    check_metrics(namespace, 'journalnode', 8081, expected_metrics)
+
+
+if __name__ == "__main__":
+    namespace: str = sys.argv[1]
+
+    log_level = "DEBUG"
+    logging.basicConfig(
+        level=log_level,
+        format="%(asctime)s %(levelname)s: %(message)s",
+        stream=sys.stdout,
+    )
+
+    check_namenode_metrics(namespace)
+    check_datanode_metrics(namespace)
+    check_journalnode_metrics(namespace)
+
+    print("All expected metrics found")