chore(nifi): Bump to 1.28.1, upgrade postgres jdbc driver

.github/workflows/dev_nifi.yaml

@@ -5,7 +5,7 @@ env:
   IMAGE_NAME: nifi
   # TODO (@NickLarsenNZ): Use a versioned image with stackable0.0.0-dev or stackableXX.X.X so that
   # the demo is reproducable for the release and it will be automatically replaced for the release branch.
-  IMAGE_VERSION: 1.27.0-postgresql
+  IMAGE_VERSION: 1.28.1-postgresql
   REGISTRY_PATH: stackable
   DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-nifi"
 

demos/demos-v2.yaml

@@ -64,6 +64,7 @@ demos:
       - s3
       - earthquakes
     manifests:
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml
@@ -85,6 +86,7 @@ demos:
       - s3
       - water-levels
     manifests:
+      - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml
       - plainYaml: https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/setup-superset.yaml

demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml

@@ -6,10 +6,30 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-druid-coordinator
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for Druid Coordinator to be created'
+              kubectl wait --for=create pod/druid-coordinator-default-0
+              echo 'Waiting for Druid Coordinator to be ready'
+              kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m
       containers:
         - name: create-druid-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor
           volumeMounts:
             - name: ingestion-job-spec
               mountPath: /tmp/ingestion-job-spec

demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml

@@ -6,10 +6,31 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-nifi
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for NiFi to be created'
+              kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m
+              echo 'Waiting for NiFi to be ready'
+              kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m
       containers:
         - name: create-nifi-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script

demos/nifi-kafka-druid-earthquake-data/serviceaccount.yaml

@@ -0,0 +1,47 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: demo-serviceaccount
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: demo-clusterrolebinding
+subjects:
+  - kind: ServiceAccount
+    name: demo-serviceaccount
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: demo-clusterrole
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: demo-clusterrole
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create

demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml

@@ -6,10 +6,31 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-superset
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for Superset to be created'
+              kubectl wait --for=create pod/superset-node-default-0 --timeout=30m
+              echo 'Waiting for Superset to be ready'
+              kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m
       containers:
         - name: setup-superset
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script

demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml

@@ -6,10 +6,32 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-druid-coordinator
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for Druid Coordinator to be created'
+              kubectl wait --for=create pod/druid-coordinator-default-0
+              echo 'Waiting for Druid Coordinator to be ready'
+              kubectl wait --for=condition=Ready pod/druid-coordinator-default-0 --timeout=30m
       containers:
         - name: create-druid-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/stations-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-compaction-job-spec.json https://druid-coordinator:8281/druid/coordinator/v1/config/compaction"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/stations-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor
+              curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor
+              curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-compaction-job-spec.json https://druid-coordinator:8281/druid/coordinator/v1/config/compaction
           volumeMounts:
             - name: ingestion-job-spec
               mountPath: /tmp/ingestion-job-spec

demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml

@@ -6,10 +6,31 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-nifi
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for NiFi to be created'
+              kubectl wait --for=create pod/nifi-node-default-0 --timeout=30m
+              echo 'Waiting for NiFi to be ready'
+              kubectl wait --for=condition=Ready pod/nifi-node-default-0 --timeout=30m
       containers:
         - name: create-nifi-ingestion-job
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.xml && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.xml
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script

demos/nifi-kafka-druid-water-level-data/serviceaccount.yaml

@@ -0,0 +1,47 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: demo-serviceaccount
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: demo-clusterrolebinding
+subjects:
+  - kind: ServiceAccount
+    name: demo-serviceaccount
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: demo-clusterrole
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: demo-clusterrole
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - batch
+    resources:
+      - jobs
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - pods/exec
+    verbs:
+      - create

demos/nifi-kafka-druid-water-level-data/setup-superset.yaml

@@ -6,10 +6,31 @@ metadata:
 spec:
   template:
     spec:
+      serviceAccountName: demo-serviceaccount
+      initContainers:
+        - name: wait-for-superset
+          image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              echo 'Waiting for Superset to be created'
+              kubectl wait --for=create pod/superset-node-default-0 --timeout=30m
+              echo 'Waiting for Superset to be ready'
+              kubectl wait --for=condition=Ready pod/superset-node-default-0 --timeout=30m
       containers:
         - name: setup-superset
           image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
-          command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip && python -u /tmp/script/script.py"]
+          command:
+            - bash
+            - -euo
+            - pipefail
+            - -c
+            - |
+              curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip
+              python -u /tmp/script/script.py
           volumeMounts:
             - name: script
               mountPath: /tmp/script

demos/signal-processing/Dockerfile-nifi

@@ -1,3 +1,13 @@
-FROM oci.stackable.tech/sdp/nifi:1.27.0-stackable0.0.0-dev
+FROM oci.stackable.tech/sdp/nifi:1.28.1-stackable0.0.0-dev
 
-RUN curl --fail -o /stackable/nifi/postgresql-42.6.0.jar "https://repo.stackable.tech/repository/misc/postgresql-timescaledb/postgresql-42.6.0.jar"
+# This is the postgresql JDBC driver from https://jdbc.postgresql.org/download/
+# There appear to be no signatures to validate against 😬
+#
+# VERSION="42.7.5"
+# curl -O "https://jdbc.postgresql.org/download/postgresql-$VERSION.jar"
+# curl --fail -u "your_username" --upload-file "postgresql-$VERSION.jar" 'https://repo.stackable.tech/repository/misc/jdbc/'
+# rm "postgresql-$VERSION.jar"
+
+# IMPORTANT (@NickLarsenNZ): Changing this version requires a change in the NiFi template (DownloadAndWriteToDB.xml)
+ARG DRIVER_VERSION="42.7.5"
+RUN curl --fail -o "/stackable/nifi/postgresql-$DRIVER_VERSION.jar" "https://repo.stackable.tech/repository/misc/jdbc/postgresql-$DRIVER_VERSION.jar"

demos/signal-processing/DownloadAndWriteToDB.xml

@@ -189,7 +189,7 @@
                 </entry>
                 <entry>
                     <key>database-driver-locations</key>
-                    <value>/stackable/nifi/postgresql-42.6.0.jar</value>
+                    <value>/stackable/nifi/postgresql-42.7.5.jar</value>
                 </entry>
                 <entry>
                     <key>kerberos-user-service</key>