Make uid/gid configurable & change group of files (#849)

lfrancke · web-flow · commit cb993ee88e3e · 2024-10-04T10:41:53.000Z
* WIP

* Fix install command

* Rework image to get the commands in order and have everything owned by stackable:0

* Make precommit happy(er)

* Adjust Druid

* Adjust Hadoop

* Update to image-tools 0.0.13

* Fix Druid &amp; Hadoop

* Fix Hadoop

* Adjust HBase

* Adjust Hello-World

* Fix Hadoop

* Fix HBase

* WIP Fix Hive

* Delete unused file
diff --git a/.github/ISSUE_TEMPLATE/update-base-java.md b/.github/ISSUE_TEMPLATE/update-base-java.md
@@ -64,7 +64,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 # Test a product image can build, eg: ZooKeeper
 bake --product zookeeper=x.y.z # where x.y.z is a valid product version using the newly added Java version
diff --git a/.github/ISSUE_TEMPLATE/update-base-stackable.md b/.github/ISSUE_TEMPLATE/update-base-stackable.md
@@ -53,7 +53,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product vector=x.y.z # where x.y.z is a valid version
 ```
diff --git a/.github/ISSUE_TEMPLATE/update-base-vector.md b/.github/ISSUE_TEMPLATE/update-base-vector.md
@@ -71,7 +71,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product vector=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-airflow.md b/.github/ISSUE_TEMPLATE/update-product-airflow.md
@@ -63,7 +63,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product airflow=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-druid.md b/.github/ISSUE_TEMPLATE/update-product-druid.md
@@ -67,7 +67,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product druid=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md b/.github/ISSUE_TEMPLATE/update-product-hbase-phoenix-omid.md
@@ -71,7 +71,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product hbase=x.y.z # where x.y.z is the new version added in this PR
 bake --product omid=x.y.z # where x.y.z is the new version added in this PR
diff --git a/.github/ISSUE_TEMPLATE/update-product-hdfs.md b/.github/ISSUE_TEMPLATE/update-product-hdfs.md
@@ -65,7 +65,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product hadoop=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-hive.md b/.github/ISSUE_TEMPLATE/update-product-hive.md
@@ -64,7 +64,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product hive=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-kafka.md b/.github/ISSUE_TEMPLATE/update-product-kafka.md
@@ -75,7 +75,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product kafka=x.y.z # where x.y.z is the new version added in this PR
 bake --product kafka-testing-tools=1.0.0 # This version doesn't change
diff --git a/.github/ISSUE_TEMPLATE/update-product-nifi.md b/.github/ISSUE_TEMPLATE/update-product-nifi.md
@@ -64,7 +64,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product nifi=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-opa.md b/.github/ISSUE_TEMPLATE/update-product-opa.md
@@ -63,7 +63,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product opa=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-spark.md b/.github/ISSUE_TEMPLATE/update-product-spark.md
@@ -65,7 +65,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product spark-k8s=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-superset.md b/.github/ISSUE_TEMPLATE/update-product-superset.md
@@ -65,7 +65,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product superset=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/.github/ISSUE_TEMPLATE/update-product-trino.md b/.github/ISSUE_TEMPLATE/update-product-trino.md
@@ -73,7 +73,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product trino=x.y.z # where x.y.z is the new version added in this PR
 bake --product trino-cli=x.y.z # where x.y.z is the new version added in this PR
diff --git a/.github/ISSUE_TEMPLATE/update-product-zookeeper.md b/.github/ISSUE_TEMPLATE/update-product-zookeeper.md
@@ -64,7 +64,7 @@ This list should be completed by the assignee(s), once respective PRs have been
 
 ```shell
 # See the latest version at https://pypi.org/project/image-tools-stackabletech/
-pip install image-tools-stackabletech==0.0.12
+pip install image-tools-stackabletech==0.0.13
 
 bake --product zookeeper=x.y.z # where x.y.z is the new version added in this PR
 
diff --git a/airflow/Dockerfile b/airflow/Dockerfile
@@ -14,7 +14,6 @@ FROM stackable/image/vector AS airflow-build-image
 ARG PRODUCT
 ARG PYTHON
 ARG TARGETARCH
-ARG TARGETOS
 
 COPY airflow/constraints-${PRODUCT}-python${PYTHON}.txt /tmp/constraints.txt
 
@@ -61,7 +60,7 @@ ARG PYTHON
 ARG RELEASE
 ARG TINI
 ARG TARGETARCH
-ARG TARGETOS
+ARG STACKABLE_USER_UID
 
 LABEL name="Apache Airflow" \
       maintainer="info@stackable.tech" \
@@ -72,51 +71,56 @@ LABEL name="Apache Airflow" \
       description="This image is deployed by the Stackable Operator for Apache Airflow."
 
 COPY airflow/licenses /licenses
-
-# Update image and install python
-RUN microdnf update && \
-    microdnf install \
-    ca-certificates \
-    cyrus-sasl \
-    git \
-    libpq \
-    openldap \
-    openldap-clients \
-    openssh-clients \
-    openssl-libs \
-    openssl-pkcs11 \
-    python${PYTHON} \
-    socat \
-    unixODBC && \
-    microdnf clean all && \
-    rm -rf /var/cache/yum
+COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/entrypoint.sh /entrypoint.sh
+COPY --chown=${STACKABLE_USER_UID}:0 airflow/stackable/utils/run-airflow.sh /run-airflow.sh
 
 ENV HOME=/stackable
 ENV AIRFLOW_USER_HOME_DIR=/stackable
 ENV PATH=$PATH:/bin:$HOME/app/bin
 ENV AIRFLOW_HOME=$HOME/airflow
 
+# Update image and install python
+RUN <<EOF
+microdnf update
+microdnf install \
+  ca-certificates \
+  cyrus-sasl \
+  git \
+  libpq \
+  openldap \
+  openldap-clients \
+  openssh-clients \
+  openssl-libs \
+  openssl-pkcs11 \
+  python${PYTHON} \
+  socat \
+  unixODBC
+microdnf clean all
+rm -rf /var/cache/yum
 
 # Get the correct `tini` binary for our architecture.
 # It is used as an init alternative in the entrypoint
-RUN mkdir -pv ${AIRFLOW_HOME} && \
-    mkdir -pv ${AIRFLOW_HOME}/dags && \
-    mkdir -pv ${AIRFLOW_HOME}/logs && \
-    chown --recursive stackable:stackable ${AIRFLOW_HOME} && \
-    curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
-
-COPY airflow/stackable/utils/entrypoint.sh /entrypoint.sh
-COPY airflow/stackable/utils/run-airflow.sh /run-airflow.sh
-RUN chmod a+x /entrypoint.sh && \
-    chmod a+x /run-airflow.sh && \
-    chmod +x /usr/bin/tini
-
-COPY --from=airflow-build-image --chown=stackable:stackable /stackable/ ${HOME}/
-COPY --from=gitsync-image --chown=stackable:stackable /git-sync /stackable/git-sync
-
-USER stackable
+curl -o /usr/bin/tini "https://repo.stackable.tech/repository/packages/tini/tini-${TINI}-${TARGETARCH}"
+chmod a+x /entrypoint.sh
+chmod a+x /run-airflow.sh
+chmod +x /usr/bin/tini
+
+mkdir -pv ${AIRFLOW_HOME}
+mkdir -pv ${AIRFLOW_HOME}/dags
+mkdir -pv ${AIRFLOW_HOME}/logs
+
+# All files and folders owned by root to support running as arbitrary users
+# This is best practice as all container users will belong to the root group (0)
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
+EOF
+
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
+COPY --from=airflow-build-image --chown=${STACKABLE_USER_UID}:0 /stackable/ ${HOME}/
+COPY --from=gitsync-image --chown=${STACKABLE_USER_UID}:0 /git-sync /stackable/git-sync
+
 ENTRYPOINT ["/usr/bin/tini", "--", "/run-airflow.sh"]
 CMD []
 
diff --git a/conf.py b/conf.py
@@ -89,3 +89,10 @@
         "ignore-error": "true",
     },
 ]
+
+args = {
+    "STACKABLE_USER_NAME": "stackable",
+    "STACKABLE_USER_UID": "1000",
+    "STACKABLE_USER_GID": "1000",
+    "DELETE_CACHES": "true"
+}
diff --git a/druid/Dockerfile b/druid/Dockerfile
@@ -8,6 +8,7 @@ ARG JACKSON_DATAFORMAT_XML
 ARG STAX2_API
 ARG WOODSTOX_CORE
 ARG AUTHORIZER
+ARG STACKABLE_USER_UID
 
 # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.)
 # This can be used to speed up builds when disk space is of no concern.
@@ -31,11 +32,11 @@ microdnf clean all
 rm -rf /var/cache/yum
 EOF
 
-USER stackable
+USER ${STACKABLE_USER_UID}
 WORKDIR /stackable
 
-COPY --chown=stackable:stackable druid/stackable/patches/apply_patches.sh /stackable/apache-druid-${PRODUCT}-src/patches/apply_patches.sh
-COPY --chown=stackable:stackable druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/patches/${PRODUCT}
+COPY --chown=stackable:0 druid/stackable/patches/apply_patches.sh /stackable/apache-druid-${PRODUCT}-src/patches/apply_patches.sh
+COPY --chown=stackable:0 druid/stackable/patches/${PRODUCT} /stackable/apache-druid-${PRODUCT}-src/patches/${PRODUCT}
 
 # Cache mounts are owned by root by default
 # We need to explicitly give the uid to use which is hardcoded to "1000" in stackable-base
@@ -45,9 +46,9 @@ COPY --chown=stackable:stackable druid/stackable/patches/${PRODUCT} /stackable/a
 # with a "directory not empty" error on the first builder to finish, as other builders
 # are still working in the cache directory.
 
-RUN --mount=type=cache,id=maven-${PRODUCT},uid=1000,target=/stackable/.m2/repository \
-    --mount=type=cache,id=npm-${PRODUCT},uid=1000,target=/stackable/.npm \
-    --mount=type=cache,id=cache-${PRODUCT},uid=1000,target=/stackable/.cache \
+RUN --mount=type=cache,id=maven-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository \
+    --mount=type=cache,id=npm-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.npm \
+    --mount=type=cache,id=cache-${PRODUCT},uid=${STACKABLE_USER_UID},target=/stackable/.cache \
     <<EOF
 curl "https://repo.stackable.tech/repository/packages/druid/apache-druid-${PRODUCT}-src.tar.gz" | tar -xzC .
 cd apache-druid-${PRODUCT}-src
@@ -79,6 +80,7 @@ FROM stackable/image/java-base AS final
 
 ARG PRODUCT
 ARG RELEASE
+ARG STACKABLE_USER_UID
 
 ARG NAME="Apache Druid"
 ARG DESCRIPTION="This image is deployed by the Stackable Operator for Apache Druid"
@@ -102,27 +104,29 @@ LABEL io.openshift.tags="ubi9,stackable,druid,sdp"
 LABEL io.k8s.description="${DESCRIPTION}"
 LABEL io.k8s.display-name="${NAME}"
 
+
+COPY --chown=${STACKABLE_USER_UID}:0 --from=druid-builder /stackable/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT}
+COPY --chown=${STACKABLE_USER_UID}:0 druid/stackable/bin /stackable/bin
+COPY --chown=${STACKABLE_USER_UID}:0 druid/licenses /licenses
+
 RUN <<EOF
 microdnf update
 microdnf clean all
 rpm -qa --qf "%{NAME}-%{VERSION}-%{RELEASE}\n" | sort > /stackable/package_manifest.txt
 rm -rf /var/cache/yum
-EOF
-
-USER stackable
-WORKDIR /stackable
-
-COPY --chown=stackable:stackable --from=druid-builder /stackable/apache-druid-${PRODUCT} /stackable/apache-druid-${PRODUCT}
-COPY --chown=stackable:stackable druid/stackable/bin /stackable/bin
-COPY --chown=stackable:stackable druid/licenses /licenses
 
-RUN <<EOF
 ln -s /stackable/apache-druid-${PRODUCT} /stackable/druid
 
 # Force to overwrite the existing 'run-druid'
 ln -sf /stackable/bin/run-druid /stackable/druid/bin/run-druid
+
+# All files and folders owned by root to support running as arbitrary users
+# This is best practice as all container users will belong to the root group (0)
+chown -R ${STACKABLE_USER_UID}:0 /stackable
+chmod -R g=u /stackable
 EOF
 
+USER ${STACKABLE_USER_UID}
 ENV PATH="${PATH}":/stackable/druid/bin
 
 WORKDIR /stackable/druid
diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile
diff --git a/hbase/Dockerfile b/hbase/Dockerfile
diff --git a/hello-world/Dockerfile b/hello-world/Dockerfile
diff --git a/hive/Dockerfile b/hive/Dockerfile
diff --git a/stackable-base/Dockerfile b/stackable-base/Dockerfile
