Chore: update demos for release (#60)

* upgrade postgres and redis versions, upgrade airflow-scheduled-job demo versions

* update versions in hbase-hdfs-load-cycling-data and airflow-scheduled-job demos

* Update logging demo for the next release

* Update signal-processing demo and container image

* upgrade minio version and nifi-kafka-druid-superset-s3 versions and druid db credentials

* anomaly-detection: update trino, superset, spark products

* docs(demos/data-lakehouse-iceberg-trino-spark): update requirements to 12 nodes

* docs(demos/data-lakehouse-iceberg-trino-spark): be less specific about how many files appear in minio

* docs(demos/data-lakehouse-iceberg-trino-spark): update the link for tpch

* docs(demos/data-lakehouse-iceberg-trino-spark): upgrade the NOTE about bug to IMPORTANT, and move it above the image.

* bump versions for jupyterhub-pyspark demo

* bump opa version for the trino-superset-s3 stack

* bump testing-tools in nifi-kafka-druid demos and revert druid version bump

* bump end-2-end-security versions

* consolidate stackable versions

* more version bumps (untested)

* bump to 24.7.0

* adapt to release 24.7

* Apply suggestions from code review

---------

Co-authored-by: Siegfried Weber <mail@siegfriedweber.net>
Co-authored-by: Techassi <sascha.lautenschlaeger@stackable.tech>
Co-authored-by: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com>
Co-authored-by: Nick Larsen <nick.larsen@stackable.tech>
Co-authored-by: Malte Sander <malte.sander.it@gmail.com>
Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com>

Author: 6 people

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-pyspark-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-date-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"]
       containers:
         - name: create-nifi-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -12,11 +12,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka -l app.kubernetes.io/instance=kafka"]
       containers:
         - name: create-spark-ingestion-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml"]
           volumeMounts:
             - name: manifest

demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-testdata
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for job load-test-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-test-data"]
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/end-to-end-security/create-spark-report.yaml

@@ -12,7 +12,7 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-trino-tables
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command:
             - bash
             - -euo
@@ -23,7 +23,7 @@ spec:
               kubectl wait --timeout=30m --for=condition=complete job/create-tables-in-trino
       containers:
         - name: create-spark-report
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command:
             - bash
             - -euo

demos/end-to-end-security/create-trino-tables.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
         - name: create-hfile-and-import-to-hbase
-          image: docker.stackable.tech/stackable/hbase:2.4.17-stackable24.3.0
+          image: docker.stackable.tech/stackable/hbase:2.4.18-stackable24.7.0
           env:
             - name: HADOOP_USER_NAME
               value: stackable

demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: distcp-cycling-data
-          image: docker.stackable.tech/stackable/hadoop:3.3.6-stackable24.3.0
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.7.0
           env:
             - name: HADOOP_USER_NAME
               value: stackable