From e7e3283ea76289c1b5312611a18b8e9a1a2c78c3 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 8 Oct 2024 17:04:37 +0200 Subject: [PATCH 1/6] Make uid/gid configurable & change group of files This is a follow-up for https://github.com/stackabletech/docker-images/pull/849 and includes: - The missing bits for Hive - Kafka --- hive/Dockerfile | 37 +++++++++++++---------- kafka/Dockerfile | 76 ++++++++++++++++++++++-------------------------- 2 files changed, 56 insertions(+), 57 deletions(-) diff --git a/hive/Dockerfile b/hive/Dockerfile index af7555d6d..1da3c13c6 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -103,40 +103,47 @@ LABEL io.openshift.tags="ubi9,stackable,hive,sdp" LABEL io.k8s.description="${DESCRIPTION}" LABEL io.k8s.display-name="${NAME}" -RUN < /stackable/package_manifest.txt -rm -rf /var/cache/yum -EOF - -USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin -RUN ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here # TODO: Do we really need all of Hadoop in here? COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP} -RUN ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop + +RUN < /stackable/package_manifest.txt +rm -rf /var/cache/yum + +ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore +ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop # The next two sections for S3 and Azure use hardcoded version numbers on purpose instead of wildcards # This way the build will fail should one of the files not be available anymore in a later Hadoop version! # Add S3 Support for Hive (support for s3a://) -RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar /stackable/hive-metastore/lib/ -RUN cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/hive-metastore/lib/ +cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-aws-${HADOOP}.jar /stackable/hive-metastore/lib/ +cp /stackable/hadoop/share/hadoop/tools/lib/aws-java-sdk-bundle-${AWS_JAVA_SDK_BUNDLE}.jar /stackable/hive-metastore/lib/ # Add Azure ABFS support (support for abfs://) -RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stackable/hive-metastore/lib/ -RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/ -RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/ +cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stackable/hive-metastore/lib/ +cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/ +cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/ + +# All files and folders owned by root to support running as arbitrary users +# This is best practice as all container users will belong to the root group (0) +chown -R ${STACKABLE_USER_UID}:0 /stackable +chmod -R g=u /stackable +EOF COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx COPY hive/licenses /licenses +USER ${STACKABLE_USER_UID} + ENV HADOOP_HOME=/stackable/hadoop ENV HIVE_HOME=/stackable/hive-metastore ENV PATH="${PATH}":/stackable/hadoop/bin:/stackable/hive-metastore/bin diff --git a/kafka/Dockerfile b/kafka/Dockerfile index e3fc02895..94b6ea409 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -9,8 +9,9 @@ ARG PRODUCT ARG SCALA ARG OPA_AUTHORIZER ARG JMX_EXPORTER +ARG STACKABLE_USER_UID -USER stackable +USER ${STACKABLE_USER_UID} WORKDIR /stackable RUN curl "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT}-src.tgz" | tar -xzC . && \ @@ -27,28 +28,12 @@ RUN curl "https://repo.stackable.tech/repository/packages/kafka/kafka-${PRODUCT} RUN curl https://repo.stackable.tech/repository/packages/kafka-opa-authorizer/opa-authorizer-${OPA_AUTHORIZER}-all.jar \ -o /stackable/kafka_${SCALA}-${PRODUCT}/libs/opa-authorizer-${OPA_AUTHORIZER}-all.jar -COPY --chown=stackable:stackable kafka/stackable/jmx/ /stackable/jmx/ +COPY --chown=${STACKABLE_USER_UID}:0 kafka/stackable/jmx/ /stackable/jmx/ RUN curl https://repo.stackable.tech/repository/packages/jmx-exporter/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar \ -o /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \ chmod +x /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar && \ ln -s /stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar /stackable/jmx/jmx_prometheus_javaagent.jar -# For earlier versions this script removes the .class file that contains the -# vulnerable code. -# TODO: This can be restricted to target only versions which do not honor the environment -# varible that has been set above but this has not currently been implemented -COPY shared/log4shell.sh /bin -RUN /bin/log4shell.sh /stackable/kafka_${SCALA}-${PRODUCT} - -# Ensure no vulnerable files are left over -# This will currently report vulnerable files being present, as it also alerts on -# SocketNode.class, which we do not remove with our scripts. -# Further investigation will be needed whether this should also be removed. -COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64 -COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64 -COPY shared/log4shell_scanner /bin/log4shell_scanner -RUN /bin/log4shell_scanner s /stackable/kafka_${SCALA}-${PRODUCT} -# === FROM stackable/image/java-base AS final @@ -56,6 +41,7 @@ ARG RELEASE ARG PRODUCT ARG SCALA ARG KCAT +ARG STACKABLE_USER_UID LABEL name="Apache Kafka" \ maintainer="info@stackable.tech" \ @@ -67,32 +53,38 @@ LABEL name="Apache Kafka" \ # This is needed for kubectl COPY kafka/kubernetes.repo /etc/yum.repos.d/kubernetes.repo -RUN microdnf update && \ - microdnf install \ - # needed by kcat for kerberos - cyrus-sasl-gssapi \ - # Can be removed once listener-operator integration is used - kubectl && \ - microdnf clean all && \ - rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt && \ - rm -rf /var/cache/yum - -USER stackable -WORKDIR /stackable - -COPY --chown=stackable:stackable kafka/licenses /licenses +COPY --chown=${STACKABLE_USER_UID}:0 kafka/licenses /licenses +COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kafka-builder /stackable/jmx/ /stackable/jmx/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses -# We copy opa-authorizer.jar and jmx-exporter through the builder image to have an absolutely minimal final image -# (e.g. we don't even need curl in it). -COPY --chown=stackable:stackable --from=kafka-builder /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka_${SCALA}-${PRODUCT} -COPY --chown=stackable:stackable --from=kafka-builder /stackable/jmx/ /stackable/jmx/ -COPY --chown=stackable:stackable --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/bin/kcat-${KCAT} -COPY --chown=stackable:stackable --from=kcat /licenses /licenses +WORKDIR /stackable -RUN ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat && \ - # kcat was located in /stackable/kcat - legacy - ln -s /stackable/bin/kcat /stackable/kcat && \ - ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka +RUN < /stackable/package_manifest.txt +rm -rf /var/cache/yum + +ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat +# kcat was located in /stackable/kcat - legacy +ln -s /stackable/bin/kcat /stackable/kcat +ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka + +# All files and folders owned by root to support running as arbitrary users +# This is best practice as all container users will belong to the root group (0) +chown -R ${STACKABLE_USER_UID}:0 /stackable +chmod -R g=u /stackable +EOF + +USER ${STACKABLE_USER_UID} ENV PATH="${PATH}:/stackable/bin:/stackable/kafka/bin" From e6606efe856dcab66f553e3ddbd15be7cb439423 Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Tue, 8 Oct 2024 17:28:06 +0200 Subject: [PATCH 2/6] More tools now migrated but not tested yet: - Kafka Testing Tools - KCat - NiFi - Omid --- kafka-testing-tools/Dockerfile | 7 ++-- kcat/Dockerfile | 3 +- nifi/Dockerfile | 71 ++++++++++++++++------------------ omid/Dockerfile | 54 ++++++++++++++------------ 4 files changed, 68 insertions(+), 67 deletions(-) diff --git a/kafka-testing-tools/Dockerfile b/kafka-testing-tools/Dockerfile index 5fdda0d13..a939b6051 100644 --- a/kafka-testing-tools/Dockerfile +++ b/kafka-testing-tools/Dockerfile @@ -8,6 +8,7 @@ FROM stackable/image/stackable-base AS final ARG PRODUCT ARG KCAT ARG RELEASE +ARG STACKABLE_USER_UID LABEL name="Kafka Testing Tools" \ maintainer="info@stackable.tech" \ @@ -29,11 +30,11 @@ RUN microdnf install \ && rm -rf /var/cache/yum # Store kcat version with binary name and add softlink -COPY --chown=stackable:stackable --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT} +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /stackable/kcat-${KCAT} RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat -COPY --chown=stackable:stackable --from=kcat /licenses /licenses +COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses -COPY --chown=stackable:stackable kafka-testing-tools/licenses /licenses +COPY --chown=${STACKABLE_USER_UID}:0 kafka-testing-tools/licenses /licenses ENTRYPOINT ["/stackable/kcat"] diff --git a/kcat/Dockerfile b/kcat/Dockerfile index 4da109e21..5cd2ecf2e 100644 --- a/kcat/Dockerfile +++ b/kcat/Dockerfile @@ -7,6 +7,7 @@ FROM stackable/image/java-base AS builder ARG PRODUCT +ARG STACKABLE_USER_UID RUN microdnf update \ && microdnf install \ @@ -32,7 +33,7 @@ RUN curl -O https://repo.stackable.tech/repository/packages/kcat/kcat-${PRODUCT} && cd kcat-${PRODUCT} \ && ./bootstrap.sh -COPY --chown=stackable:stackable kcat/licenses /licenses +COPY --chown=${STACKABLE_USER_UID}:0 kcat/licenses /licenses # SNIPPET 1 # 145.2 gcc -I/stackable/kcat-1.7.0/tmp-bootstrap/usr/include -I/stackable/kcat-1.7.0/tmp-bootstrap/usr/include -g -O2 -Wall -Wsign-compare -Wfloat-equal -Wpointer-arith -Wcast-align -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -Wl,-rpath-link=/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib -Wl,-rpath-link=/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib kcat.o format.o tools.o input.o json.o avro.o -o kcat -lm -ldl -lpthread -lrt -lpthread -lrt -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libavro.a /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libjansson.a -lcurl /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libserdes.a -Wl,-Bstatic -lavro -Wl,-Bdynamic /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libyajl_s.a -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib //stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/librdkafka.a -lm -ldl -lpthread -lrt -lz -lcrypto -lssl -lsasl2 -lm -ldl -lpthread -lrt -lpthread -lrt -L/stackable/kcat-1.7.0/tmp-bootstrap/usr/lib /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libavro.a /stackable/kcat-1.7.0/tmp-bootstrap/usr/lib/libjansson.a -lcurl diff --git a/nifi/Dockerfile b/nifi/Dockerfile index c5e5325da..f614b96b4 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -5,6 +5,7 @@ FROM stackable/image/java-devel AS nifi-builder ARG PRODUCT ARG MAVEN_VERSION="3.9.8" +ARG STACKABLE_USER_UID RUN microdnf update && \ microdnf clean all && \ @@ -22,10 +23,10 @@ RUN if [[ "${PRODUCT}" == 2.* ]] ; then \ ln -sf /tmp/apache-maven-${MAVEN_VERSION}/bin/mvn /usr/bin/mvn ; \ fi -USER stackable +USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:stackable nifi/stackable/patches /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/patches /stackable/patches # NOTE: NiFi 1.21.0 source build does not work with the current arm64 git runners due to java heap issues: # @@ -82,28 +83,11 @@ RUN if [[ "${PRODUCT}" == "1.21.0" ]] ; then \ rm -rf /stackable/nifi-${PRODUCT}/docs ; \ fi -# === -# For earlier versions this script removes the .class file that contains the -# vulnerable code. -# TODO: This can be restricted to target only versions which do not honor the environment -# varible that has been set above but this has not currently been implemented -COPY shared/log4shell.sh /bin -RUN /bin/log4shell.sh /stackable/nifi-${PRODUCT} - -# Ensure no vulnerable files are left over -# This will currently report vulnerable files being present, as it also alerts on -# SocketNode.class, which we do not remove with our scripts. -# Further investigation will be needed whether this should also be removed. -COPY shared/log4shell_1.6.1-log4shell_Linux_x86_64 /bin/log4shell_scanner_x86_64 -COPY shared/log4shell_1.6.1-log4shell_Linux_aarch64 /bin/log4shell_scanner_aarch64 -COPY shared/log4shell_scanner /bin/log4shell_scanner -RUN /bin/log4shell_scanner s /stackable/nifi-${PRODUCT} -# === - FROM stackable/image/java-base AS final ARG PRODUCT ARG RELEASE +ARG STACKABLE_USER_UID LABEL name="Apache NiFi" \ maintainer="info@stackable.tech" \ @@ -113,28 +97,39 @@ LABEL name="Apache NiFi" \ summary="The Stackable image for Apache NiFi." \ description="This image is deployed by the Stackable Operator for Apache NiFi." -RUN microdnf update && \ - microdnf install \ - # Required to install nipyapi - python-pip && \ - microdnf clean all && \ - rm -rf /var/cache/yum && \ - # The nipyapi is required for the ReportingTaskJob - pip install --no-cache-dir nipyapi==0.19.1 && \ - # For backwards compatibility we create a softlink in /bin where the jar used to be as long as we are root - # This can be removed once older versions / operators using this are no longer supported - ln -s /stackable/stackable-bcrypt.jar /bin/stackable-bcrypt.jar +COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/nifi-${PRODUCT} /stackable/nifi-${PRODUCT}/ +COPY --chown=${STACKABLE_USER_UID}:0 --from=nifi-builder /stackable/stackable-bcrypt.jar /stackable/stackable-bcrypt.jar + +COPY --chown=${STACKABLE_USER_UID}:0 nifi/stackable/bin /stackable/bin +COPY --chown=${STACKABLE_USER_UID}:0 nifi/licenses /licenses +COPY --chown=${STACKABLE_USER_UID}:0 nifi/python /stackable/python + +RUN < Date: Wed, 9 Oct 2024 13:07:38 +0200 Subject: [PATCH 3/6] - OPA - Spark (WIP) --- opa/Dockerfile | 37 +++++++++++++++++++++++-------------- spark-k8s/Dockerfile | 37 +++++++++++++++++++------------------ 2 files changed, 42 insertions(+), 32 deletions(-) diff --git a/opa/Dockerfile b/opa/Dockerfile index 6420ad5f2..c91c5642e 100644 --- a/opa/Dockerfile +++ b/opa/Dockerfile @@ -86,32 +86,41 @@ FROM stackable/image/vector ARG PRODUCT ARG RELEASE +ARG STACKABLE_USER_UID LABEL name="Open Policy Agent" \ maintainer="info@stackable.tech" \ vendor="Stackable GmbH" \ version="${PRODUCT}" \ release="${RELEASE}" \ - summary="The Stackable image for OPA." \ + summary="The Stackable image for Open Policy Agent (OPA)." \ description="This image is deployed by the Stackable Operator for OPA." -RUN microdnf update && \ - microdnf install \ - # Required for filtering logs - jq && \ - microdnf clean all && \ - rm -rf /var/cache/yum - COPY opa/licenses /licenses -USER stackable -WORKDIR /stackable/opa +COPY --from=opa-builder --chown=${STACKABLE_USER_UID}:0 /opa/opa /stackable/opa/opa +COPY --from=opa-bundle-builder --chown=${STACKABLE_USER_UID}:0 /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder +COPY --from=multilog-builder --chown=${STACKABLE_USER_UID}:0 /daemontools/admin/daemontools/command/multilog /stackable/multilog -COPY --from=opa-builder /opa/opa /stackable/opa/opa -COPY --from=opa-bundle-builder --chown=stackable:stackable /opa-bundle-builder/target/release/stackable-opa-bundle-builder /stackable/opa-bundle-builder -COPY --from=multilog-builder --chown=stackable:stackable /daemontools/admin/daemontools/command/multilog /stackable/multilog +COPY --chown=${STACKABLE_USER_UID}:0 opa/stackable/bin /stackable/opa/bin -COPY --chown=stackable:stackable opa/stackable/bin /stackable/opa/bin +RUN < Date: Fri, 11 Oct 2024 09:37:58 +0200 Subject: [PATCH 4/6] Adds Spark and a changelog entry --- CHANGELOG.md | 3 ++ kafka-testing-tools/Dockerfile | 1 - spark-k8s/Dockerfile | 57 +++++++++++++++++++--------------- 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bd17dc544..00c150389 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,7 @@ All notable changes to this project will be documented in this file. README file ([#819]). - Update cargo-cyclonedx to 0.5.5 and build CycloneDX 1.5 files ([#783]) - Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]) +- Make username, user id, group id configurable, use numeric ids everywhere, change group of all files to 0 ([#849], [#890]) ### Removed @@ -63,6 +64,7 @@ All notable changes to this project will be documented in this file. [#822]: https://github.com/stackabletech/docker-images/pull/822 [#846]: https://github.com/stackabletech/docker-images/pull/846 [#848]: https://github.com/stackabletech/docker-images/pull/848 +[#849]: https://github.com/stackabletech/docker-images/pull/849 [#851]: https://github.com/stackabletech/docker-images/pull/851 [#852]: https://github.com/stackabletech/docker-images/pull/852 [#853]: https://github.com/stackabletech/docker-images/pull/853 @@ -76,6 +78,7 @@ All notable changes to this project will be documented in this file. [#879]: https://github.com/stackabletech/docker-images/pull/879 [#881]: https://github.com/stackabletech/docker-images/pull/881 [#882]: https://github.com/stackabletech/docker-images/pull/882 +[#890]: https://github.com/stackabletech/docker-images/pull/890 ## [24.7.0] - 2024-07-24 diff --git a/kafka-testing-tools/Dockerfile b/kafka-testing-tools/Dockerfile index a939b6051..a15cc7d9d 100644 --- a/kafka-testing-tools/Dockerfile +++ b/kafka-testing-tools/Dockerfile @@ -34,7 +34,6 @@ COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /stackable/kcat-${KCAT}/kcat /s RUN ln -s /stackable/kcat-${KCAT} /stackable/kcat COPY --chown=${STACKABLE_USER_UID}:0 --from=kcat /licenses /licenses - COPY --chown=${STACKABLE_USER_UID}:0 kafka-testing-tools/licenses /licenses ENTRYPOINT ["/stackable/kcat"] diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index 3e60d2402..c269dcfd4 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -172,6 +172,7 @@ ARG WOODSTOX_CORE ARG JMX_EXPORTER ARG TARGETARCH ARG TINI +ARG STACKABLE_USER_UID WORKDIR /stackable/spark-${PRODUCT} @@ -283,24 +284,6 @@ LABEL name="Apache Spark" \ summary="The Stackable image for Apache Spark with PySpark support." \ description="This image is deployed by the Stackable Operator for Apache Spark on Kubernetes." -RUN microdnf update && \ - microdnf install \ - gzip \ - hostname \ - # required for spark startup scripts - procps \ - "python${PYTHON}" \ - "python${PYTHON}-pip" \ - zip \ - # This is needed by the Spark UI to display process information using jps and jmap - # Copying the binaries from the builder stage failed. - "java-${JAVA_VERSION}-openjdk-devel" \ - && microdnf clean all \ - && rm -rf /var/cache/yum - -RUN ln -s /usr/bin/python${PYTHON} /usr/bin/python \ - && ln -s /usr/bin/pip-${PYTHON} /usr/bin/pip - ENV HOME=/stackable ENV SPARK_HOME=/stackable/spark @@ -308,20 +291,44 @@ ENV PATH=$SPARK_HOME:$PATH:/bin:$JAVA_HOME/bin:$JAVA_HOME/jre/bin:$HOME/.local/b ENV PYSPARK_PYTHON=/usr/bin/python ENV PYTHONPATH=$SPARK_HOME/python +COPY spark-k8s/stackable /stackable +COPY spark-k8s/licenses /licenses + COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-${PRODUCT}/dist /stackable/spark COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/spark-${PRODUCT}/assembly/target/bom.json /stackable/spark/spark-${PRODUCT}.cdx.json COPY --chown=${STACKABLE_USER_UID}:0 --from=spark-builder /stackable/jmx /stackable/jmx COPY --from=spark-builder /usr/bin/tini /usr/bin/tini -RUN ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/jmx/jmx_prometheus_javaagent.jar \ - # Symlink example jar, so that we can easily use it in tests - && ln -s /stackable/spark/examples/jars/spark-examples_*.jar /stackable/spark/examples/jars/spark-examples.jar +RUN < Date: Wed, 16 Oct 2024 10:12:12 +0200 Subject: [PATCH 5/6] Update CHANGELOG.md Co-authored-by: Nick <10092581+NickLarsenNZ@users.noreply.github.com> --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffdb02d98..d204cc377 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,7 +31,7 @@ All notable changes to this project will be documented in this file. - Enable [Docker build checks](https://docs.docker.com/build/checks/) ([#872]). - java: migrate to temurin jdk/jre ([#894]). - tools: bump kubectl to `1.31.1` and jq to `1.7.1` ([#896]). -- Make username, user id, group id configurable, use numeric ids everywhere, change group of all files to 0 ([#849], [#890]) +- Make username, user id, group id configurable, use numeric ids everywhere, change group of all files to 0 ([#849], [#890]). ### Removed From 4411137fa9d043c3d77d40593b47c7dce949bfbd Mon Sep 17 00:00:00 2001 From: Lars Francke Date: Wed, 16 Oct 2024 10:53:38 +0200 Subject: [PATCH 6/6] Update comment --- druid/Dockerfile | 4 ++-- hadoop/Dockerfile | 4 ++-- hbase/Dockerfile | 4 ++-- hello-world/Dockerfile | 4 ++-- hive/Dockerfile | 4 ++-- kafka/Dockerfile | 4 ++-- nifi/Dockerfile | 4 ++-- omid/Dockerfile | 4 ++-- opa/Dockerfile | 4 ++-- spark-k8s/Dockerfile | 4 ++-- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/druid/Dockerfile b/druid/Dockerfile index 104597bdd..a294ac552 100644 --- a/druid/Dockerfile +++ b/druid/Dockerfile @@ -120,8 +120,8 @@ ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid # Force to overwrite the existing 'run-druid' ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index a0ecc328d..124a7abc4 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -169,8 +169,8 @@ find . -name 'hadoop-*tests.jar' -type f -delete # It is so non-root users (as we are) can mount a FUSE device and let other users access it echo "user_allow_other" > /etc/fuse.conf -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/hbase/Dockerfile b/hbase/Dockerfile index 31e5cc478..a8228dec7 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -354,8 +354,8 @@ ln --symbolic --logical --verbose "/stackable/hbase-${PRODUCT}" /stackable/hbase ln --symbolic --logical --verbose "/stackable/hbase-operator-tools-${HBASE_OPERATOR_TOOLS}" /stackable/hbase-operator-tools ln --symbolic --logical --verbose "/stackable/phoenix/phoenix-server-hbase-${HBASE_PROFILE}.jar" "/stackable/hbase/lib/phoenix-server-hbase-${HBASE_PROFILE}.jar" -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/hello-world/Dockerfile b/hello-world/Dockerfile index 89caabf03..9680043b6 100644 --- a/hello-world/Dockerfile +++ b/hello-world/Dockerfile @@ -22,8 +22,8 @@ rm -rf /var/cache/yum curl "https://repo.stackable.tech/repository/packages/hello-world/hello-world-${PRODUCT}.jar" -o /stackable/hello-world.jar -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/hive/Dockerfile b/hive/Dockerfile index 1da3c13c6..794faef6b 100644 --- a/hive/Dockerfile +++ b/hive/Dockerfile @@ -133,8 +133,8 @@ cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stackabl cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/ cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/ -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/kafka/Dockerfile b/kafka/Dockerfile index 94b6ea409..4d7b204c0 100644 --- a/kafka/Dockerfile +++ b/kafka/Dockerfile @@ -78,8 +78,8 @@ ln -s /stackable/bin/kcat-${KCAT} /stackable/bin/kcat ln -s /stackable/bin/kcat /stackable/kcat ln -s /stackable/kafka_${SCALA}-${PRODUCT} /stackable/kafka -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/nifi/Dockerfile b/nifi/Dockerfile index f614b96b4..472be39a5 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -123,8 +123,8 @@ pip install --no-cache-dir nipyapi==0.19.1 && \ # This can be removed once older versions / operators using this are no longer supported ln -s /stackable/stackable-bcrypt.jar /bin/stackable-bcrypt.jar -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/omid/Dockerfile b/omid/Dockerfile index cc4ee35e9..1c7d14f82 100644 --- a/omid/Dockerfile +++ b/omid/Dockerfile @@ -95,8 +95,8 @@ rm /stackable/omid-tso-server/conf/hbase-site.xml # To support arbitrary user ids on OpenShift, this folder must belong to the root group. mkdir /stackable/logs -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/opa/Dockerfile b/opa/Dockerfile index 4a8ca5b70..df7695c97 100644 --- a/opa/Dockerfile +++ b/opa/Dockerfile @@ -112,8 +112,8 @@ microdnf install \ microdnf clean all rm -rf /var/cache/yum -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF diff --git a/spark-k8s/Dockerfile b/spark-k8s/Dockerfile index c269dcfd4..965efdaf3 100644 --- a/spark-k8s/Dockerfile +++ b/spark-k8s/Dockerfile @@ -322,8 +322,8 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j # Symlink example jar, so that we can easily use it in tests ln -s /stackable/spark/examples/jars/spark-examples_*.jar /stackable/spark/examples/jars/spark-examples.jar -# All files and folders owned by root to support running as arbitrary users -# This is best practice as all container users will belong to the root group (0) +# All files and folders owned by root group to support running as arbitrary users. +# This is best practice as all container users will belong to the root group (0). chown -R ${STACKABLE_USER_UID}:0 /stackable chmod -R g=u /stackable EOF