From 1b1947e7ac8cecf5b77bd7fefe26276bfa295031 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 10 Sep 2024 20:37:27 +0200 Subject: [PATCH 01/15] WIP --- airflow/Dockerfile | 81 +++++++++++++++++++++------------------ conf.py | 6 +++ stackable-base/Dockerfile | 19 +++++---- 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 6c30adc66..0b685634c 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -4,7 +4,7 @@ ARG GIT_SYNC # For updated versions check https://github.com/kubernetes/git-sync/releases # which should contain a image location (e.g. registry.k8s.io/git-sync/git-sync:v3.6.8) -FROM oci.stackable.tech/sdp/git-sync:${GIT_SYNC} as gitsync-image +FROM oci.stackable.tech/sdp/git-sync:${GIT_SYNC} AS gitsync-image FROM stackable/image/statsd_exporter AS statsd_exporter-builder @@ -60,9 +60,10 @@ ARG RELEASE ARG TINI ARG TARGETARCH ARG TARGETOS +ARG STACKABLE_USER_UID LABEL name="Apache Airflow" \ - maintainer="info@stackable.tech" \ + maintainer="info@stackastatsd_exporble.tech" \ vendor="Stackable GmbH" \ version="${PRODUCT}" \ release="${RELEASE}" \ @@ -71,50 +72,56 @@ LABEL name="Apache Airflow" \ COPY airflow/licenses /licenses +COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh +COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh + # Update image and install python -RUN microdnf update && \ - microdnf install \ - ca-certificates \ - cyrus-sasl \ - git \ - libpq \ - openldap \ - openldap-clients \ - openssh-clients \ - openssl-libs \ - openssl-pkcs11 \ - python${PYTHON} \ - socat \ - unixODBC && \ - microdnf clean all && \ - rm -rf /var/cache/yum +RUN <> /stackable/.profile -chown stackable:stackable /stackable/.bashrc -chown stackable:stackable /stackable/.profile +chown ${STACKABLE_USER_UID}:0 /stackable/.bashrc +chown ${STACKABLE_USER_UID}:0 /stackable/.profile # CVE-2023-37920: Remove "e-Tugra" root certificates # e-Tugra's root certificates were subject to an investigation prompted by reporting of security issues in their systems @@ -156,8 +159,8 @@ if [ "$(trust list --filter=ca-anchors | grep -c 'E-Tugra')" != "0" ]; then fi EOF -COPY --from=product-utils-builder --chown=stackable:stackable /config-utils/target/release/config-utils /stackable/config-utils -COPY --from=product-utils-builder --chown=stackable:stackable /config-utils/config-utils.cdx.xml /stackable/config-utils.cdx.xml +COPY --from=product-utils-builder --chown=${STACKABLE_USER_UID}:0 /config-utils/target/release/config-utils /stackable/config-utils +COPY --from=product-utils-builder --chown=${STACKABLE_USER_UID}:0 /config-utils/config-utils.cdx.xml /stackable/config-utils.cdx.xml ENV PATH="${PATH}:/stackable" # These labels have mostly been superceded by the OpenContainer spec annotations below but it doesn't hurt to include them From 6e8b5c3123611270d952dbb0a995be84a6270db8 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 11 Sep 2024 23:45:00 +0200 Subject: [PATCH 02/15] Fix install command --- airflow/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 0b685634c..c1aef5dd2 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -117,11 +117,16 @@ RUN < Date: Thu, 12 Sep 2024 00:41:05 +0200 Subject: [PATCH 03/15] Rework image to get the commands in order and have everything owned by stackable:0 --- airflow/Dockerfile | 58 ++++++++++++++++++++-------------------------- 1 file changed, 25 insertions(+), 33 deletions(-) diff --git a/airflow/Dockerfile b/airflow/Dockerfile index c1aef5dd2..a63e8b37f 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -13,7 +13,6 @@ FROM stackable/image/vector AS airflow-build-image ARG PRODUCT ARG PYTHON ARG TARGETARCH -ARG TARGETOS COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt @@ -59,11 +58,10 @@ ARG PYTHON ARG RELEASE ARG TINI ARG TARGETARCH -ARG TARGETOS ARG STACKABLE_USER_UID LABEL name="Apache Airflow" \ - maintainer="info@stackastatsd_exporble.tech" \ + maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ version="${PRODUCT}" \ release="${RELEASE}" \ @@ -71,25 +69,29 @@ LABEL name="Apache Airflow" \ description="This image is deployed by the Stackable Operator for Apache Airflow." COPY airflow/licenses /licenses - COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh +ENV HOME=/stackable +ENV AIRFLOW_USER_HOME_DIR=/stackable +ENV PATH=$PATH:/bin:$HOME/app/bin +ENV AIRFLOW_HOME=$HOME/airflow + # Update image and install python RUN < Date: Thu, 12 Sep 2024 08:09:58 +0200 Subject: [PATCH 04/15] Make precommit happy(er) --- conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf.py b/conf.py index 201ea0ea6..329e83b71 100644 --- a/conf.py +++ b/conf.py @@ -93,5 +93,5 @@ args = { "STACKABLE_USER_NAME": "stackable", "STACKABLE_USER_UID": "1000", - "STACKABLE_USER_GID": "1000" + "STACKABLE_USER_GID": "1000", } From 0b0701960697163c913dabe8fc1cfebc82be5ff5 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Thu, 12 Sep 2024 18:57:09 +0200 Subject: [PATCH 05/15] Adjust Druid --- airflow/Dockerfile | 2 +- conf.py | 1 + druid/Dockerfile | 33 ++++++++++++++++++--------------- stackable-base/Dockerfile | 1 + 4 files changed, 21 insertions(+), 16 deletions(-) diff --git a/airflow/Dockerfile b/airflow/Dockerfile index a63e8b37f..70ce0ff7b 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -108,7 +108,7 @@ mkdir -pv ${AIRFLOW_HOME}/dags mkdir -pv ${AIRFLOW_HOME}/logs # All files and folders owned by root to support running as arbitrary users -# This is best practice as all users will belong to group `0` +# This is best practice as all container users will belong to the root group (0) chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/conf.py b/conf.py index 329e83b71..23c5e54ce 100644 --- a/conf.py +++ b/conf.py @@ -94,4 +94,5 @@ "STACKABLE_USER_NAME": "stackable", "STACKABLE_USER_UID": "1000", "STACKABLE_USER_GID": "1000", + "DELETE_CACHES": "true" } diff --git a/druid/Dockerfile b/druid/Dockerfile index 62e0456eb..ec4ef68da 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -30,11 +30,11 @@ microdnf clean all rm -rf /var/cache/yum EOF -USER stackable +USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:stackable druid/stackable/patches/apply_patches.sh /stackable/apache-druid-${PRODUCT}-src/patches/apply_patches.sh -COPY --chown=stackable:stackable druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/patches/${PRODUCT} +COPY --chown=stackable:0 druid/stackable/patches/apply_patches.sh /stackable/apache-druid-${PRODUCT}-src/patches/apply_patches.sh +COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/patches/${PRODUCT} # Cache mounts are owned by root by default # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base @@ -44,9 +44,9 @@ COPY --chown=stackable:stackable druid/stackable/patches/${PRODUCT} /stackable/a # with a "directory not empty" error on the first builder to finish, as other builders # are still working in the cache directory. -RUN --mount=type=cache,id=maven-${PRODUCT},uid=1000,target=/stackable/.m2/repository \ - --mount=type=cache,id=npm-${PRODUCT},uid=1000,target=/stackable/.npm \ - --mount=type=cache,id=cache-${PRODUCT},uid=1000,target=/stackable/.cache \ +RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository \ + --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \ + --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \ < /stackable/package_manifest.txt rm -rf /var/cache/yum -EOF - -USER stackable -WORKDIR /stackable - -COPY --chown=stackable:stackable --from=druid-builder /stackable/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT} -COPY --chown=stackable:stackable druid/stackable/bin /stackable/bin -COPY --chown=stackable:stackable druid/licenses /licenses -RUN < Date: Thu, 12 Sep 2024 19:03:11 +0200 Subject: [PATCH 06/15] Adjust Hadoop --- hadoop/Dockerfile | 78 ++++++++++++++++++++++++++--------------------- 1 file changed, 44 insertions(+), 34 deletions(-) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 98dfa7c02..f2f0cf07d 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -9,6 +9,7 @@ ARG PROTOBUF ARG TARGETARCH ARG TARGETOS +USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY hadoop/stackable/jmx /stackable/jmx @@ -92,6 +93,7 @@ FROM stackable/image/java-devel AS hdfs-utils-builder ARG HDFS_UTILS ARG PRODUCT +USER ${STACKABLE_USER_UID} WORKDIR /stackable # The Stackable HDFS utils contain an OPA authorizer, group mapper & topology provider. @@ -120,35 +122,58 @@ LABEL name="Apache Hadoop" \ summary="The Stackable image for Apache Hadoop." \ description="This image is deployed by the Stackable Operator for Apache Hadoop / HDFS." +COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/jmx /stackable/jmx/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/async-profiler /stackable/async-profiler/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar +COPY hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin + + # fuse is required for fusermount (called by fuse_dfs) # fuse-libs is required for fuse_dfs (not included in fuse) # openssl -> not sure -RUN microdnf update && \ - microdnf install \ - fuse \ - fuse-libs \ - # tar is required for `kubectl cp` which can be used to copy the log files - # or profiler flamegraph from the Pod - tar && \ - microdnf clean all && \ - rm -rf /var/cache/yum +RUN < /etc/fuse.conf +echo "user_allow_other" > /etc/fuse.conf -USER stackable -WORKDIR /stackable +# All files and folders owned by root to support running as arbitrary users +# This is best practice as all container users will belong to the root group (0) +chown -R ${STACKABLE_USER_UID}:0 /stackable +chmod -R g=u /stackable +EOF -COPY --chown=stackable:stackable --from=builder /stackable/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT}/ -COPY --chown=stackable:stackable --from=builder /stackable/jmx /stackable/jmx/ -COPY --chown=stackable:stackable --from=builder /stackable/async-profiler /stackable/async-profiler/ -COPY --chown=stackable:stackable --from=hdfs-utils-builder /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar -RUN ln -s /stackable/hadoop-${PRODUCT} /stackable/hadoop +COPY hadoop/licenses /licenses -COPY hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin +USER stackable ENV HOME=/stackable ENV LD_LIBRARY_PATH=/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server @@ -164,20 +189,5 @@ ENV ASYNC_PROFILER_HOME=/stackable/async-profiler ENV HADOOP_YARN_HOME=/stackable/hadoop ENV HADOOP_MAPRED_HOME=/stackable/hadoop -# Remove unneeded binaries: -# - code sources -# - mapreduce/yarn binaries that were built as cross-project dependencies -# - minicluster (only used for testing) and test .jars -# - json-io: this is a transitive dependency pulled in by cedarsoft/java-utils/json-io and is excluded in 3.4.0. See CVE-2023-34610. -RUN rm -rf /stackable/hadoop/share/hadoop/common/sources/ && \ - rm -rf /stackable/hadoop/share/hadoop/hdfs/sources/ && \ - rm -rf /stackable/hadoop/share/hadoop/tools/sources/ && \ - rm -rf /stackable/hadoop/share/hadoop/tools/lib/json-io-*.jar && \ - rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-mapreduce-client-*.jar && \ - rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-yarn-server*.jar && \ - find . -name 'hadoop-minicluster-*.jar' -type f -delete && \ - find . -name 'hadoop-client-minicluster-*.jar' -type f -delete && \ - find . -name 'hadoop-*tests.jar' -type f -delete - WORKDIR /stackable/hadoop CMD ["echo", "This image is not meant to be 'run' directly."] From fc18df66c09e1ecc42a08da909a9ad190e2200ec Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Fri, 13 Sep 2024 09:34:06 +0200 Subject: [PATCH 07/15] Update to image-tools 0.0.13 --- .github/ISSUE_TEMPLATE/update-base-java.md | 2 +- .github/ISSUE_TEMPLATE/update-base-stackable.md | 2 +- .github/ISSUE_TEMPLATE/update-base-vector.md | 2 +- .github/ISSUE_TEMPLATE/update-product-airflow.md | 2 +- .github/ISSUE_TEMPLATE/update-product-druid.md | 2 +- .github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md | 2 +- .github/ISSUE_TEMPLATE/update-product-hdfs.md | 2 +- .github/ISSUE_TEMPLATE/update-product-hive.md | 2 +- .github/ISSUE_TEMPLATE/update-product-kafka.md | 2 +- .github/ISSUE_TEMPLATE/update-product-nifi.md | 2 +- .github/ISSUE_TEMPLATE/update-product-opa.md | 2 +- .github/ISSUE_TEMPLATE/update-product-spark.md | 2 +- .github/ISSUE_TEMPLATE/update-product-superset.md | 2 +- .github/ISSUE_TEMPLATE/update-product-trino.md | 2 +- .github/ISSUE_TEMPLATE/update-product-zookeeper.md | 2 +- .github/actions/build-product-image/action.yml | 2 +- .github/workflows/preflight.yaml | 2 +- .github/workflows/release.yml | 2 +- 18 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/update-base-java.md b/.github/ISSUE_TEMPLATE/update-base-java.md index 60a57c176..12682597f 100644 --- a/.github/ISSUE_TEMPLATE/update-base-java.md +++ b/.github/ISSUE_TEMPLATE/update-base-java.md @@ -58,7 +58,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 # Test a product image can build, eg: ZooKeeper bake --product zookeeper=x.y.z # where x.y.z is a valid product version using the newly added Java version diff --git a/.github/ISSUE_TEMPLATE/update-base-stackable.md b/.github/ISSUE_TEMPLATE/update-base-stackable.md index 60e5b288e..98602bbab 100644 --- a/.github/ISSUE_TEMPLATE/update-base-stackable.md +++ b/.github/ISSUE_TEMPLATE/update-base-stackable.md @@ -41,7 +41,7 @@ Part of #xxx. ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product vector=x.y.z # where x.y.z is a valid version ``` diff --git a/.github/ISSUE_TEMPLATE/update-base-vector.md b/.github/ISSUE_TEMPLATE/update-base-vector.md index a591836a5..04d8b3613 100644 --- a/.github/ISSUE_TEMPLATE/update-base-vector.md +++ b/.github/ISSUE_TEMPLATE/update-base-vector.md @@ -56,7 +56,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product vector=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-airflow.md b/.github/ISSUE_TEMPLATE/update-product-airflow.md index 4b2b1a565..eb176d27c 100644 --- a/.github/ISSUE_TEMPLATE/update-product-airflow.md +++ b/.github/ISSUE_TEMPLATE/update-product-airflow.md @@ -49,7 +49,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product airflow=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-druid.md b/.github/ISSUE_TEMPLATE/update-product-druid.md index c3b497039..da7c541c7 100644 --- a/.github/ISSUE_TEMPLATE/update-product-druid.md +++ b/.github/ISSUE_TEMPLATE/update-product-druid.md @@ -51,7 +51,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product druid=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md b/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md index 9bf2069eb..849c8ae81 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md +++ b/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md @@ -57,7 +57,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product hbase=x.y.z # where x.y.z is the new version added in this PR bake --product omid=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hdfs.md b/.github/ISSUE_TEMPLATE/update-product-hdfs.md index 50c75854a..38d7a8422 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hdfs.md +++ b/.github/ISSUE_TEMPLATE/update-product-hdfs.md @@ -51,7 +51,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product hadoop=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hive.md b/.github/ISSUE_TEMPLATE/update-product-hive.md index aa24e1eea..73d123a77 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hive.md +++ b/.github/ISSUE_TEMPLATE/update-product-hive.md @@ -50,7 +50,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product hive=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-kafka.md b/.github/ISSUE_TEMPLATE/update-product-kafka.md index 1055fc301..b68946be6 100644 --- a/.github/ISSUE_TEMPLATE/update-product-kafka.md +++ b/.github/ISSUE_TEMPLATE/update-product-kafka.md @@ -61,7 +61,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product kafka=x.y.z # where x.y.z is the new version added in this PR bake --product kafka-testing-tools=1.0.0 # This version doesn't change diff --git a/.github/ISSUE_TEMPLATE/update-product-nifi.md b/.github/ISSUE_TEMPLATE/update-product-nifi.md index 5a4949444..86194e432 100644 --- a/.github/ISSUE_TEMPLATE/update-product-nifi.md +++ b/.github/ISSUE_TEMPLATE/update-product-nifi.md @@ -50,7 +50,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product nifi=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-opa.md b/.github/ISSUE_TEMPLATE/update-product-opa.md index f20dcb6b3..ca8756a0f 100644 --- a/.github/ISSUE_TEMPLATE/update-product-opa.md +++ b/.github/ISSUE_TEMPLATE/update-product-opa.md @@ -49,7 +49,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product opa=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-spark.md b/.github/ISSUE_TEMPLATE/update-product-spark.md index 0e4b0c550..8c0f7431c 100644 --- a/.github/ISSUE_TEMPLATE/update-product-spark.md +++ b/.github/ISSUE_TEMPLATE/update-product-spark.md @@ -50,7 +50,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product spark-k8s=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-superset.md b/.github/ISSUE_TEMPLATE/update-product-superset.md index 5cfcec15f..6b9629c74 100644 --- a/.github/ISSUE_TEMPLATE/update-product-superset.md +++ b/.github/ISSUE_TEMPLATE/update-product-superset.md @@ -50,7 +50,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product superset=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-trino.md b/.github/ISSUE_TEMPLATE/update-product-trino.md index 65b0132b2..0bd88100b 100644 --- a/.github/ISSUE_TEMPLATE/update-product-trino.md +++ b/.github/ISSUE_TEMPLATE/update-product-trino.md @@ -59,7 +59,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product trino=x.y.z # where x.y.z is the new version added in this PR bake --product trino-cli=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-zookeeper.md b/.github/ISSUE_TEMPLATE/update-product-zookeeper.md index a665b602b..7f2736814 100644 --- a/.github/ISSUE_TEMPLATE/update-product-zookeeper.md +++ b/.github/ISSUE_TEMPLATE/update-product-zookeeper.md @@ -50,7 +50,7 @@ description? ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product zookeeper=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/actions/build-product-image/action.yml b/.github/actions/build-product-image/action.yml index 7033d7686..91f20c2c8 100644 --- a/.github/actions/build-product-image/action.yml +++ b/.github/actions/build-product-image/action.yml @@ -10,7 +10,7 @@ inputs: required: true image-tools-version: description: The Stackable image-tools version - default: 0.0.12 + default: 0.0.13 build-cache-username: description: Build cache username default: github diff --git a/.github/workflows/preflight.yaml b/.github/workflows/preflight.yaml index 4eff89436..59686c970 100644 --- a/.github/workflows/preflight.yaml +++ b/.github/workflows/preflight.yaml @@ -70,7 +70,7 @@ jobs: - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 with: python-version: '3.x' - - run: pip install image-tools-stackabletech==0.0.12 + - run: pip install image-tools-stackabletech==0.0.13 - name: Install preflight run: | wget https://github.com/redhat-openshift-ecosystem/openshift-preflight/releases/download/1.10.0/preflight-linux-amd64 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 47a36783c..c2f921625 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -79,7 +79,7 @@ jobs: - name: Set up syft uses: anchore/sbom-action/download-syft@e8d2a6937ecead383dfe75190d104edd1f9c5751 # v0.16.0 - name: Install image-tools-stackabletech - run: pip install image-tools-stackabletech==0.0.12 + run: pip install image-tools-stackabletech==0.0.13 - uses: docker/login-action@0d4c9c5ea7693da7b068278f7b52bda2a190a446 # v3.2.0 with: registry: docker.stackable.tech From 5c0b829f5f6d90b4ddfdfb5e09bfb502dd69026a Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Mon, 16 Sep 2024 16:31:51 +0200 Subject: [PATCH 08/15] Fix Druid & Hadoop --- druid/Dockerfile | 1 + hadoop/Dockerfile | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index ec4ef68da..7c61b8126 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -7,6 +7,7 @@ ARG JACKSON_DATAFORMAT_XML ARG STAX2_API ARG WOODSTOX_CORE ARG AUTHORIZER +ARG STACKABLE_USER_UID # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) # This can be used to speed up builds when disk space is of no concern. diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index f2f0cf07d..8602c9e60 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -8,6 +8,7 @@ ARG JMX_EXPORTER ARG PROTOBUF ARG TARGETARCH ARG TARGETOS +ARG STACKABLE_USER_UID USER ${STACKABLE_USER_UID} WORKDIR /stackable @@ -92,6 +93,7 @@ FROM stackable/image/java-devel AS hdfs-utils-builder ARG HDFS_UTILS ARG PRODUCT +ARG STACKABLE_USER_UID USER ${STACKABLE_USER_UID} WORKDIR /stackable @@ -113,6 +115,7 @@ FROM stackable/image/java-base AS final ARG PRODUCT ARG RELEASE ARG HDFS_UTILS +ARG STACKABLE_USER_UID LABEL name="Apache Hadoop" \ maintainer="info@stackable.tech" \ @@ -173,7 +176,7 @@ EOF COPY hadoop/licenses /licenses -USER stackable +USER ${STACKABLE_USER_UID} ENV HOME=/stackable ENV LD_LIBRARY_PATH=/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server From 42476b2e84fb151cfd1dc6b9281491cd2249a0cf Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Mon, 16 Sep 2024 17:03:17 +0200 Subject: [PATCH 09/15] Fix Hadoop --- hadoop/Dockerfile | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 8602c9e60..c7ef7fd1a 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -10,11 +10,10 @@ ARG TARGETARCH ARG TARGETOS ARG STACKABLE_USER_UID -USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY hadoop/stackable/jmx /stackable/jmx -COPY hadoop/stackable/fuse_dfs_wrapper /stackable/fuse_dfs_wrapper +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/jmx /stackable/jmx +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /stackable/fuse_dfs_wrapper # The symlink from JMX Exporter 0.16.1 to the versionless link exists because old HDFS Operators (up until and including 23.7) used to hardcode # the version of JMX Exporter like this: "-javaagent:/stackable/jmx/jmx_prometheus_javaagent-0.16.1.jar" @@ -55,7 +54,7 @@ RUN microdnf update && \ WORKDIR /stackable -COPY hadoop/stackable/patches /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/patches /stackable/patches # Hadoop Pipes requires libtirpc to build, whose headers are not packaged in RedHat UBI, so skip building this module # Build from source to enable FUSE module, and to apply custom patches. @@ -129,7 +128,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/hadoop-${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/jmx /stackable/jmx/ COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/async-profiler /stackable/async-profiler/ COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar -COPY hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin # fuse is required for fusermount (called by fuse_dfs) From 922f420111cd8301774d28458dbba6cce6eb1c4f Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Mon, 16 Sep 2024 17:06:58 +0200 Subject: [PATCH 10/15] Adjust HBase --- hbase/Dockerfile | 68 +++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/hbase/Dockerfile b/hbase/Dockerfile index d66fcc38d..549f06abe 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -14,6 +14,7 @@ ARG JMX_EXPORTER ARG HADOOP ARG TARGETARCH ARG TARGETOS +ARG STACKABLE_USER_UID # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) # This can be used to speed up builds when disk space is of no concern. @@ -21,14 +22,14 @@ ARG DELETE_CACHES="true" COPY hbase/licenses /licenses -USER stackable +USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:stackable hbase/stackable/patches /stackable/patches -COPY --chown=stackable:stackable hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx # Cache mounts are owned by root by default -# We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base +# We need to explicitly give the uid to use # And every cache needs its own id, we can't share them between stages because we might delete the caches # at the end of a run while other stages are still using it. # While this might work in theory it didn't in practice (FileNotFound exceptions etc.) @@ -38,7 +39,7 @@ COPY --chown=stackable:stackable hbase/stackable/jmx/config${JMX_EXPORTER} /stac # builder containers will share the same cache and the `rm -rf` commands will fail # with a "directory not empty" error on the first builder to finish, as other builders # are still working in the cache directory. -RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=1000,target=/stackable/.m2/repository < Date: Mon, 16 Sep 2024 17:08:33 +0200 Subject: [PATCH 11/15] Adjust Hello-World --- hello-world/Dockerfile | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/hello-world/Dockerfile b/hello-world/Dockerfile index e19ad8340..8d66520bc 100644 --- a/hello-world/Dockerfile +++ b/hello-world/Dockerfile @@ -4,6 +4,7 @@ FROM stackable/image/java-base ARG PRODUCT ARG RELEASE +ARG STACKABLE_USER_UID LABEL name="Hello World" \ maintainer="info@stackable.tech" \ @@ -13,13 +14,20 @@ LABEL name="Hello World" \ summary="The Stackable image for the Stackable Hello World Operator" \ description="This image is deployed by the Stackable Hello World Operator." -RUN microdnf update && \ - microdnf clean all && \ - rm -rf /var/cache/yum +RUN < Date: Tue, 17 Sep 2024 08:40:15 +0200 Subject: [PATCH 12/15] Fix Hadoop --- hadoop/Dockerfile | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index c7ef7fd1a..d7351f3a7 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -128,7 +128,7 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/hadoop-${PRODUCT} COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/jmx /stackable/jmx/ COPY --chown=${STACKABLE_USER_UID}:0 --from=builder /stackable/async-profiler /stackable/async-profiler/ COPY --chown=${STACKABLE_USER_UID}:0 --from=hdfs-utils-builder /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar -COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin +COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /stackable/ # fuse is required for fusermount (called by fuse_dfs) @@ -136,17 +136,18 @@ COPY --chown=${STACKABLE_USER_UID}:0 hadoop/stackable/fuse_dfs_wrapper /stackabl # openssl -> not sure RUN < Date: Tue, 17 Sep 2024 09:28:18 +0200 Subject: [PATCH 13/15] Fix HBase --- hbase/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/hbase/Dockerfile b/hbase/Dockerfile index 549f06abe..9f462651e 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -287,6 +287,7 @@ ARG HADOOP ARG PHOENIX ARG HBASE_PROFILE ARG HBASE_OPERATOR_TOOLS +ARG STACKABLE_USER_UID ARG NAME="Apache HBase" ARG DESCRIPTION="This image is deployed by the Stackable Operator for Apache HBase" From 682a9458ac7e1d39ddf45451b9d0c9affac03d6f Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 17 Sep 2024 10:32:27 +0200 Subject: [PATCH 14/15] WIP Fix Hive --- hive/Dockerfile | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/hive/Dockerfile b/hive/Dockerfile index 7efa76945..fb43aa521 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -12,19 +12,20 @@ FROM stackable/image/java-devel AS hive-builder ARG PRODUCT ARG HADOOP ARG JMX_EXPORTER +ARG STACKABLE_USER_UID # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) # This can be used to speed up builds when disk space is of no concern. ARG DELETE_CACHES="true" -COPY --chown=stackable:stackable hive/stackable /stackable +COPY --chown=${STACKABLE_USER_UID}:0 hive/stackable /stackable -USER stackable +USER ${STACKABLE_USER_UID} WORKDIR /stackable # Cache mounts are owned by root by default -# We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base -RUN --mount=type=cache,id=maven-hive,uid=1000,target=/stackable/.m2/repository < /stackable/package_manif rm -rf /var/cache/yum EOF -USER stackable +USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:stackable --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin +COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin RUN ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here # TODO: Do we really need all of Hadoop in here? -COPY --chown=stackable:stackable --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP} +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP} RUN ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop # The next two sections for S3 and Azure use hardcoded version numbers on purpose instead of wildcards @@ -131,7 +133,7 @@ RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stac RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/ RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/ -COPY --chown=stackable:stackable --from=hive-builder /stackable/jmx /stackable/jmx +COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx COPY hive/licenses /licenses ENV HADOOP_HOME=/stackable/hadoop From f1315256ca03b34f7b920a284d60b473b4fbdb14 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 2 Oct 2024 16:27:40 +0200 Subject: [PATCH 15/15] Delete unused file --- .github/actions/build-product-image/action.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .github/actions/build-product-image/action.yml diff --git a/.github/actions/build-product-image/action.yml b/.github/actions/build-product-image/action.yml deleted file mode 100644 index e69de29bb..000000000