diff --git a/resources/apt_upgrade_images.sh b/resources/apt_upgrade_images.sh deleted file mode 100755 index e80c0e73f..000000000 --- a/resources/apt_upgrade_images.sh +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2024 CS Group -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Run "apt update && apt upgrade -y" in docker images that are used in our ci/cd - -set -euo pipefail -set -x - -images=\ -"python:3.11.7-slim-bookworm "\ -"jupyter/minimal-notebook:latest "\ -"quay.io/jupyter/base-notebook:hub-4.1.5"\ - -# Target images will be: -# ghcr.io/rs-python/python:3.11.7-slim-bookworm -# ghcr.io/rs-python/jupyter/minimal-notebook:latest -# ghcr.io/rs-python/quay.io/jupyter/base-notebook:hub-4.1.5 - -dockerdir="/tmp/dockerfile" -dockerfile="$dockerdir/Dockerfile" -mkdir -p "$dockerdir" -cd "$dockerdir" - -# For each docker image -for image in $images; do - - # Save the default user in the image - user=$(docker run --rm --entrypoint whoami "$image") - - # Add our hosting github organization to the docker image - target="ghcr.io/rs-python/$image" - - # Create a tmp Dockerfile that pulls and update the image. - cat << EOF > "$dockerfile" -FROM $image -USER root -RUN apt update && apt upgrade -y -USER $user - -# Upgrade pip version -RUN pip install -U pip - -# Set labels based on the Open Containers Initiative (OCI): -# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys -# -LABEL org.opencontainers.image.source="https://github.com/RS-PYTHON/rs-server" -LABEL org.opencontainers.image.ref.name="$target" -LABEL dockerfile.url="https://github.com/RS-PYTHON/rs-server/blob/develop/resources/apt_upgrade_images.sh" - -# Note: don't remove cache so the child images that use this one as a base will build faster -# RUN rm -rf /var/cache/apt/archives /var/lib/apt/lists/* -EOF - - # For the jupyter images - if [[ $image == *"jupyter"* ]]; then - - DASK_TAG=2024.5.2 - DASK_GATEWAY_TAG=2024.1.0 - PREFECT_TAG=3.2.13 - PREFECT_DASK_TAG=0.3.3 - - cat << EOF >> "$dockerfile" - -# Install python 3.11.7 using conda then prefect and dask and other packages. -# The versions must be the same than the cluster images. -RUN conda install --yes conda-forge::python="3.11.7" - -# Note: put s3fs before boto3 to have a recent version -RUN pip install \ - dask[complete]=="${DASK_TAG}" \ - distributed=="${DASK_TAG}" \ - dask-gateway=="${DASK_GATEWAY_TAG}" \ - prefect[aws]=="${PREFECT_TAG}" \ - prefect-dask=="${PREFECT_DASK_TAG}" \ - ipywidgets \ - s3fs \ - boto3 - -# Install dot and clean conda -USER root -RUN apt install -y python3-pydot graphviz -RUN conda clean --all --yes -USER jovyan -EOF - fi - - cat "$dockerfile" - - # Build the docker image - docker build --progress plain -f "$dockerfile" -t "$target" "$dockerdir" - - # Push the docker image to the registry, if the --push option is specified. - if [[ " $@ " == *" --push "* ]]; then - docker login https://ghcr.io/v2/rs-python - docker push "$target" - fi -done diff --git a/resources/build_base_images.sh b/resources/build_base_images.sh new file mode 100755 index 000000000..ffed26b7e --- /dev/null +++ b/resources/build_base_images.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Copyright 2025 CS Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Build the base Docker images that are used in the cluster and in the ci/cd. + +set -euo pipefail +#set -x + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BUILD_DIR="$(realpath $SCRIPT_DIR/build_base_images)" + +# For each dockerfile and associated docker image name, separated by a ; +for params in \ + "Dockerfile.python;python:3.11.7-slim-bookworm" \ + "Dockerfile.jupyter;quay.io/jupyter/base-notebook:hub-5.2.1" +do + dockerfile=$(echo $params | cut -d ";" -f 1) + base=$(echo $params | cut -d ";" -f 2) + + # Add our hosting github organization to the docker image + target="ghcr.io/rs-python/$base" + + # Build the docker image + docker build \ + --build-arg BASE=${base} \ + --build-arg DASK_TAG=2024.5.2 \ + --build-arg DASK_GATEWAY_TAG=2024.1.0 \ + --build-arg PREFECT_TAG=3.2.13 \ + --build-arg PREFECT_DASK_TAG=0.3.3 \ + --progress plain \ + -f "${BUILD_DIR}/${dockerfile}" \ + -t "$target" \ + "${BUILD_DIR}" + + # Push the docker image to the registry, if the --push option is specified. + if [[ " $@ " == *" --push "* ]]; then + docker login https://ghcr.io/v2/rs-python + docker push "$target" + fi +done diff --git a/resources/build_base_images/Dockerfile.jupyter b/resources/build_base_images/Dockerfile.jupyter new file mode 100644 index 000000000..db6a7b6ec --- /dev/null +++ b/resources/build_base_images/Dockerfile.jupyter @@ -0,0 +1,89 @@ +#!/usr/bin/env bash +# Copyright 2025 CS Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# quay.io/jupyter/base-notebook: +ARG BASE=xxx +FROM ${BASE} + +ENV PATH=/home/jovyan/.local/bin:${PATH} + +ARG DASK_TAG=xxx +ARG DASK_GATEWAY_TAG=xxx +ARG PREFECT_TAG=xxx +ARG PREFECT_DASK_TAG=xxx + +COPY ./layer-cleanup.sh /usr/local/bin/ + +USER root +RUN chmod 755 /usr/local/bin/layer-cleanup.sh +RUN apt update && apt upgrade -y && layer-cleanup.sh +USER jovyan + +# Upgrade pip version +RUN pip install -U pip && layer-cleanup.sh + +# Additional python kernel to install to run the notebooks +ARG PYTHON_VERSION=3.11.7 +ARG KERNEL_NAME=py3.11.7 + +# Install additional python kernel +RUN conda create -y -n "$KERNEL_NAME" python="$PYTHON_VERSION" && \ + source activate "$KERNEL_NAME" && \ + python -V && \ + pip install -U pip && \ + pip install ipykernel && \ + layer-cleanup.sh +USER root +RUN source activate "$KERNEL_NAME" && \ + python -m ipykernel install --name "$KERNEL_NAME" --display-name "$KERNEL_NAME" && \ + layer-cleanup.sh +USER jovyan + +# Set default kernel... does not seem to work +RUN jupyter notebook --generate-config && \ + layer-cleanup.sh && \ + echo "c.MappingKernelManager.default_kernel_name='$KERNEL_NAME'" >> /home/jovyan/.jupyter/jupyter_notebook_config.py + +# Note: put s3fs before boto3 to have a recent version +RUN source activate "$KERNEL_NAME" && \ + pip install \ + dask[complete]=="${DASK_TAG}" \ + distributed=="${DASK_TAG}" \ + dask-gateway=="${DASK_GATEWAY_TAG}" \ + prefect[aws]=="${PREFECT_TAG}" \ + prefect-dask=="${PREFECT_DASK_TAG}" \ + ipywidgets \ + s3fs \ + boto3 && \ + layer-cleanup.sh + +# Install dot and clean conda +USER root +RUN apt update && apt install -y python3-pydot graphviz && layer-cleanup.sh +USER jovyan + +# Use default python version from bash session +RUN echo -e "\nsource activate $KERNEL_NAME" >> /home/jovyan/.bashrc + +USER root +RUN chown -R jovyan /home/jovyan +USER jovyan + +# Set labels based on the Open Containers Initiative (OCI): +# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys +# +LABEL org.opencontainers.image.source="https://github.com/RS-PYTHON/rs-server" +LABEL org.opencontainers.image.ref.name="ghcr.io/rs-python/quay.io/jupyter/base-notebook:hub-5.2.1" +LABEL dockerfile.url="https://github.com/RS-PYTHON/rs-server/blob/develop/resources/build_base_images/Dockerfile.jupyter.k8s" diff --git a/resources/build_base_images/Dockerfile.python b/resources/build_base_images/Dockerfile.python new file mode 100644 index 000000000..0abafe449 --- /dev/null +++ b/resources/build_base_images/Dockerfile.python @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copyright 2025 CS Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# python: +ARG BASE=xxx +FROM ${BASE} +RUN apt update && apt upgrade -y + +# Upgrade pip version +RUN pip install -U pip + +# Note: don't remove cache so the child images that use this one as a base will build faster +# RUN rm -rf /var/cache/apt/archives /var/lib/apt/lists/* + +# Set labels based on the Open Containers Initiative (OCI): +# https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys +# +LABEL org.opencontainers.image.source="https://github.com/RS-PYTHON/rs-server" +LABEL org.opencontainers.image.ref.name="ghcr.io/rs-python/python:3.11.7-slim-bookworm" +LABEL dockerfile.url="https://github.com/RS-PYTHON/rs-server/blob/develop/resources/build_base_images/Dockerfile.python" diff --git a/resources/build_base_images/layer-cleanup.sh b/resources/build_base_images/layer-cleanup.sh new file mode 100644 index 000000000..c39fc23dc --- /dev/null +++ b/resources/build_base_images/layer-cleanup.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Copyright 2024 CS Group +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apt-get autoclean --yes +apt-get autoremove --yes + +rm -rf /var/lib/apt/lists/* +# rm -rf /etc/apt/sources.list.d/* # don't remove the repo list +rm -rf /usr/local/src/* + +rm -rf /var/cache/apt/* +rm -rf /root/.cache/* +rm -rf /home/*/.cache/* +# including /root/.cache/pip +rm -rf /usr/local/share/.cache/* +# including /usr/local/share/.cache/yarn + +rm -rf /tmp/* /var/tmp/* + +conda clean --all --yes + +exit 0 diff --git a/resources/update_dask_eopf_prefect.sh b/resources/update_dask_eopf_prefect.sh index 2f32f094e..43a380093 100755 --- a/resources/update_dask_eopf_prefect.sh +++ b/resources/update_dask_eopf_prefect.sh @@ -63,7 +63,7 @@ all_files+=($e $f) # [local mode] [cluster mode] [jupyter base image] # [ghcr.io/rs-python/jupyter/minimal-notebook] [ghcr.io/rs-python/quay.io/jupyter/base-notebook] -g=$(_realpath rs-server/resources/apt_upgrade_images.sh) # + re-run with --push +g=$(_realpath rs-server/resources/build_base_images.sh) # + re-run with --push all_files+=($g) # [local mode] [jupyter with rs-client-libraries] [ghcr.io/rs-python/jupyter/rs-client-libraries/local]