diff --git a/.github/ISSUE_TEMPLATE/update-base-java.md b/.github/ISSUE_TEMPLATE/update-base-java.md index f7a7c12d4..1f24ba384 100644 --- a/.github/ISSUE_TEMPLATE/update-base-java.md +++ b/.github/ISSUE_TEMPLATE/update-base-java.md @@ -64,7 +64,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 # Test a product image can build, eg: ZooKeeper bake --product zookeeper=x.y.z # where x.y.z is a valid product version using the newly added Java version diff --git a/.github/ISSUE_TEMPLATE/update-base-stackable.md b/.github/ISSUE_TEMPLATE/update-base-stackable.md index 0b84d395d..934ed9ea1 100644 --- a/.github/ISSUE_TEMPLATE/update-base-stackable.md +++ b/.github/ISSUE_TEMPLATE/update-base-stackable.md @@ -53,7 +53,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product vector=x.y.z # where x.y.z is a valid version ``` diff --git a/.github/ISSUE_TEMPLATE/update-base-vector.md b/.github/ISSUE_TEMPLATE/update-base-vector.md index 805f2c97e..41ba7c3f3 100644 --- a/.github/ISSUE_TEMPLATE/update-base-vector.md +++ b/.github/ISSUE_TEMPLATE/update-base-vector.md @@ -71,7 +71,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product vector=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-airflow.md b/.github/ISSUE_TEMPLATE/update-product-airflow.md index c6e6d5296..ddac5bc08 100644 --- a/.github/ISSUE_TEMPLATE/update-product-airflow.md +++ b/.github/ISSUE_TEMPLATE/update-product-airflow.md @@ -63,7 +63,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product airflow=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-druid.md b/.github/ISSUE_TEMPLATE/update-product-druid.md index 521348487..7838dae55 100644 --- a/.github/ISSUE_TEMPLATE/update-product-druid.md +++ b/.github/ISSUE_TEMPLATE/update-product-druid.md @@ -67,7 +67,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product druid=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md b/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md index 705c8dae9..ce859357f 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md +++ b/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md @@ -71,7 +71,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product hbase=x.y.z # where x.y.z is the new version added in this PR bake --product omid=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hdfs.md b/.github/ISSUE_TEMPLATE/update-product-hdfs.md index e08ff92a9..25341bcc1 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hdfs.md +++ b/.github/ISSUE_TEMPLATE/update-product-hdfs.md @@ -65,7 +65,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product hadoop=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-hive.md b/.github/ISSUE_TEMPLATE/update-product-hive.md index 3efb128e9..396ccd54b 100644 --- a/.github/ISSUE_TEMPLATE/update-product-hive.md +++ b/.github/ISSUE_TEMPLATE/update-product-hive.md @@ -64,7 +64,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product hive=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-kafka.md b/.github/ISSUE_TEMPLATE/update-product-kafka.md index e4990c033..c82e7ac2a 100644 --- a/.github/ISSUE_TEMPLATE/update-product-kafka.md +++ b/.github/ISSUE_TEMPLATE/update-product-kafka.md @@ -75,7 +75,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product kafka=x.y.z # where x.y.z is the new version added in this PR bake --product kafka-testing-tools=1.0.0 # This version doesn't change diff --git a/.github/ISSUE_TEMPLATE/update-product-nifi.md b/.github/ISSUE_TEMPLATE/update-product-nifi.md index 684749ba4..ab21533c9 100644 --- a/.github/ISSUE_TEMPLATE/update-product-nifi.md +++ b/.github/ISSUE_TEMPLATE/update-product-nifi.md @@ -64,7 +64,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product nifi=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-opa.md b/.github/ISSUE_TEMPLATE/update-product-opa.md index d2de6b86b..a3d6e3bf7 100644 --- a/.github/ISSUE_TEMPLATE/update-product-opa.md +++ b/.github/ISSUE_TEMPLATE/update-product-opa.md @@ -63,7 +63,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product opa=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-spark.md b/.github/ISSUE_TEMPLATE/update-product-spark.md index 2e0aad603..94be2c3fa 100644 --- a/.github/ISSUE_TEMPLATE/update-product-spark.md +++ b/.github/ISSUE_TEMPLATE/update-product-spark.md @@ -65,7 +65,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product spark-k8s=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-superset.md b/.github/ISSUE_TEMPLATE/update-product-superset.md index 0c84c6ebb..514843a1f 100644 --- a/.github/ISSUE_TEMPLATE/update-product-superset.md +++ b/.github/ISSUE_TEMPLATE/update-product-superset.md @@ -65,7 +65,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product superset=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-trino.md b/.github/ISSUE_TEMPLATE/update-product-trino.md index 6dedd62a9..7de87f14b 100644 --- a/.github/ISSUE_TEMPLATE/update-product-trino.md +++ b/.github/ISSUE_TEMPLATE/update-product-trino.md @@ -73,7 +73,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product trino=x.y.z # where x.y.z is the new version added in this PR bake --product trino-cli=x.y.z # where x.y.z is the new version added in this PR diff --git a/.github/ISSUE_TEMPLATE/update-product-zookeeper.md b/.github/ISSUE_TEMPLATE/update-product-zookeeper.md index 5f72aff9f..6ece8ea2c 100644 --- a/.github/ISSUE_TEMPLATE/update-product-zookeeper.md +++ b/.github/ISSUE_TEMPLATE/update-product-zookeeper.md @@ -64,7 +64,7 @@ This list should be completed by the assignee(s), once respective PRs have been ```shell # See the latest version at https://pypi.org/project/image-tools-stackabletech/ -pip install image-tools-stackabletech==0.0.12 +pip install image-tools-stackabletech==0.0.13 bake --product zookeeper=x.y.z # where x.y.z is the new version added in this PR diff --git a/airflow/Dockerfile b/airflow/Dockerfile index 54b7c840a..aa5ef9f60 100644 --- a/airflow/Dockerfile +++ b/airflow/Dockerfile @@ -14,7 +14,6 @@ FROM stackable/image/vector AS airflow-build-image ARG PRODUCT ARG PYTHON ARG TARGETARCH -ARG TARGETOS COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt @@ -61,7 +60,7 @@ ARG PYTHON ARG RELEASE ARG TINI ARG TARGETARCH -ARG TARGETOS +ARG STACKABLE_USER_UID LABEL name="Apache Airflow" \ maintainer="info@stackable.tech" \ @@ -72,51 +71,56 @@ LABEL name="Apache Airflow" \ description="This image is deployed by the Stackable Operator for Apache Airflow." COPY airflow/licenses /licenses - -# Update image and install python -RUN microdnf update && \ - microdnf install \ - ca-certificates \ - cyrus-sasl \ - git \ - libpq \ - openldap \ - openldap-clients \ - openssh-clients \ - openssl-libs \ - openssl-pkcs11 \ - python${PYTHON} \ - socat \ - unixODBC && \ - microdnf clean all && \ - rm -rf /var/cache/yum +COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh +COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh ENV HOME=/stackable ENV AIRFLOW_USER_HOME_DIR=/stackable ENV PATH=$PATH:/bin:$HOME/app/bin ENV AIRFLOW_HOME=$HOME/airflow +# Update image and install python +RUN < /stackable/package_manifest.txt rm -rf /var/cache/yum -EOF - -USER stackable -WORKDIR /stackable - -COPY --chown=stackable:stackable --from=druid-builder /stackable/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT} -COPY --chown=stackable:stackable druid/stackable/bin /stackable/bin -COPY --chown=stackable:stackable druid/licenses /licenses -RUN < not sure -RUN microdnf update && \ - microdnf install \ - fuse \ - fuse-libs \ - # tar is required for `kubectl cp` which can be used to copy the log files - # or profiler flamegraph from the Pod - tar && \ - microdnf clean all && \ - rm -rf /var/cache/yum +RUN < /etc/fuse.conf +echo "user_allow_other" > /etc/fuse.conf -USER stackable -WORKDIR /stackable +# All files and folders owned by root to support running as arbitrary users +# This is best practice as all container users will belong to the root group (0) +chown -R ${STACKABLE_USER_UID}:0 /stackable +chmod -R g=u /stackable +EOF -COPY --chown=stackable:stackable --from=builder /stackable/hadoop-${PRODUCT} /stackable/hadoop-${PRODUCT}/ -COPY --chown=stackable:stackable --from=builder /stackable/jmx /stackable/jmx/ -COPY --chown=stackable:stackable --from=builder /stackable/async-profiler /stackable/async-profiler/ -COPY --chown=stackable:stackable --from=hdfs-utils-builder /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar /stackable/hadoop-${PRODUCT}/share/hadoop/common/lib/hdfs-utils-${HDFS_UTILS}.jar -RUN ln -s /stackable/hadoop-${PRODUCT} /stackable/hadoop +COPY hadoop/licenses /licenses -COPY hadoop/stackable/fuse_dfs_wrapper /stackable/hadoop/bin +USER ${STACKABLE_USER_UID} ENV HOME=/stackable ENV LD_LIBRARY_PATH=/stackable/hadoop/lib/native:/usr/lib/jvm/jre/lib/server @@ -165,20 +193,5 @@ ENV ASYNC_PROFILER_HOME=/stackable/async-profiler ENV HADOOP_YARN_HOME=/stackable/hadoop ENV HADOOP_MAPRED_HOME=/stackable/hadoop -# Remove unneeded binaries: -# - code sources -# - mapreduce/yarn binaries that were built as cross-project dependencies -# - minicluster (only used for testing) and test .jars -# - json-io: this is a transitive dependency pulled in by cedarsoft/java-utils/json-io and is excluded in 3.4.0. See CVE-2023-34610. -RUN rm -rf /stackable/hadoop/share/hadoop/common/sources/ && \ - rm -rf /stackable/hadoop/share/hadoop/hdfs/sources/ && \ - rm -rf /stackable/hadoop/share/hadoop/tools/sources/ && \ - rm -rf /stackable/hadoop/share/hadoop/tools/lib/json-io-*.jar && \ - rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-mapreduce-client-*.jar && \ - rm -rf /stackable/hadoop/share/hadoop/tools/lib/hadoop-yarn-server*.jar && \ - find . -name 'hadoop-minicluster-*.jar' -type f -delete && \ - find . -name 'hadoop-client-minicluster-*.jar' -type f -delete && \ - find . -name 'hadoop-*tests.jar' -type f -delete - WORKDIR /stackable/hadoop CMD ["echo", "This image is not meant to be 'run' directly."] diff --git a/hbase/Dockerfile b/hbase/Dockerfile index b0e550804..31e5cc478 100644 --- a/hbase/Dockerfile +++ b/hbase/Dockerfile @@ -15,6 +15,7 @@ ARG JMX_EXPORTER ARG HADOOP ARG TARGETARCH ARG TARGETOS +ARG STACKABLE_USER_UID # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) # This can be used to speed up builds when disk space is of no concern. @@ -22,14 +23,14 @@ ARG DELETE_CACHES="true" COPY hbase/licenses /licenses -USER stackable +USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:stackable hbase/stackable/patches /stackable/patches -COPY --chown=stackable:stackable hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/patches /stackable/patches +COPY --chown=${STACKABLE_USER_UID}:0 hbase/stackable/jmx/config${JMX_EXPORTER} /stackable/jmx # Cache mounts are owned by root by default -# We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base +# We need to explicitly give the uid to use # And every cache needs its own id, we can't share them between stages because we might delete the caches # at the end of a run while other stages are still using it. # While this might work in theory it didn't in practice (FileNotFound exceptions etc.) @@ -39,7 +40,7 @@ COPY --chown=stackable:stackable hbase/stackable/jmx/config${JMX_EXPORTER} /stac # builder containers will share the same cache and the `rm -rf` commands will fail # with a "directory not empty" error on the first builder to finish, as other builders # are still working in the cache directory. -RUN --mount=type=cache,id=maven-hbase-${PRODUCT},uid=1000,target=/stackable/.m2/repository < /stackable/package_manif rm -rf /var/cache/yum EOF -USER stackable +USER ${STACKABLE_USER_UID} WORKDIR /stackable -COPY --chown=stackable:stackable --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin +COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/apache-hive-metastore-${PRODUCT}-bin RUN ln -s /stackable/apache-hive-metastore-${PRODUCT}-bin /stackable/hive-metastore # It is useful to see which version of Hadoop is used at a glance # Therefore the use of the full name here # TODO: Do we really need all of Hadoop in here? -COPY --chown=stackable:stackable --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP} +COPY --chown=${STACKABLE_USER_UID}:0 --from=hadoop-builder /stackable/hadoop /stackable/hadoop-${HADOOP} RUN ln -s /stackable/hadoop-${HADOOP} /stackable/hadoop # The next two sections for S3 and Azure use hardcoded version numbers on purpose instead of wildcards @@ -132,7 +134,7 @@ RUN cp /stackable/hadoop/share/hadoop/tools/lib/hadoop-azure-${HADOOP}.jar /stac RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-storage-${AZURE_STORAGE}.jar /stackable/hive-metastore/lib/ RUN cp /stackable/hadoop/share/hadoop/tools/lib/azure-keyvault-core-${AZURE_KEYVAULT_CORE}.jar /stackable/hive-metastore/lib/ -COPY --chown=stackable:stackable --from=hive-builder /stackable/jmx /stackable/jmx +COPY --chown=${STACKABLE_USER_UID}:0 --from=hive-builder /stackable/jmx /stackable/jmx COPY hive/licenses /licenses ENV HADOOP_HOME=/stackable/hadoop diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile index becab29e0..3833d8d35 100644 --- a/stackable-base/Dockerfile +++ b/stackable-base/Dockerfile @@ -47,6 +47,9 @@ FROM registry.access.redhat.com/ubi9/ubi-minimal@sha256:c0e70387664f30cd9cf2795b # intentionally unused ARG PRODUCT +ARG STACKABLE_USER_UID +ARG STACKABLE_USER_GID +ARG STACKABLE_USER_NAME # Sets the default shell to Bash with strict error handling and robust pipeline processing. # "-e": Exits immediately if a command exits with a non-zero status @@ -110,21 +113,22 @@ microdnf install \ ### # Added only temporarily to create the user and group, removed again below microdnf install shadow-utils -groupadd --gid 1000 --system stackable +groupadd --gid ${STACKABLE_USER_GID} --system ${STACKABLE_USER_NAME} # The --no-log-init is required to work around a bug/problem in Go/Docker when very large UIDs are used # See https://github.com/moby/moby/issues/5419#issuecomment-41478290 for more context # Making this a system user prevents a mail dir from being created, expiry of passwords etc. but it will warn: # useradd warning: stackable's uid 1000 is greater than SYS_UID_MAX 999 # We can safely ignore this warning, to get rid of the warning we could change /etc/login.defs but that does not seem worth it +# We'll leave the home directory hardcoded to /stackable because I don't want to deal with which chars might be valid and which might not in user name vs. directory useradd \ --no-log-init \ - --gid stackable \ - --uid 1000 \ + --gid ${STACKABLE_USER_GID} \ + --uid ${STACKABLE_USER_UID} \ --system \ --create-home \ --home-dir /stackable \ - stackable + ${STACKABLE_USER_NAME} microdnf remove shadow-utils microdnf clean all @@ -140,8 +144,8 @@ microdnf clean all echo -e "if [ -f ~/.bashrc ]; then\n\tsource ~/.bashrc\nfi" >> /stackable/.profile -chown stackable:stackable /stackable/.bashrc -chown stackable:stackable /stackable/.profile +chown ${STACKABLE_USER_UID}:0 /stackable/.bashrc +chown ${STACKABLE_USER_UID}:0 /stackable/.profile cp /root/.curlrc /stackable/.curlrc chown stackable:0 /stackable/.curlrc @@ -167,8 +171,8 @@ if [ "$(trust list --filter=ca-anchors | grep -c 'E-Tugra')" != "0" ]; then fi EOF -COPY --from=product-utils-builder --chown=stackable:stackable /config-utils/target/release/config-utils /stackable/config-utils -COPY --from=product-utils-builder --chown=stackable:stackable /config-utils/config-utils_bin.cdx.xml /stackable/config-utils.cdx.xml +COPY --from=product-utils-builder --chown=${STACKABLE_USER_UID}:0 /config-utils/target/release/config-utils /stackable/config-utils +COPY --from=product-utils-builder --chown=${STACKABLE_USER_UID}:0 /config-utils/config-utils_bin.cdx.xml /stackable/config-utils.cdx.xml ENV PATH="${PATH}:/stackable" # These labels have mostly been superceded by the OpenContainer spec annotations below but it doesn't hurt to include them