From 5f751d36c19d7c870182f3e6157e02c3bdc5311e Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Sat, 1 Feb 2025 09:38:26 +0530 Subject: [PATCH 01/29] Dockerfiles added --- docker/Dockerfile.dask-worker | 41 +++++++++++ docker/Dockerfile.jupyterhub | 41 +++++++++++ docker/Dockerfile.jupyterlab | 70 +++++++++++++++++++ docker/Dockerfile.workflow-controller | 61 ++++++++++++++++ docker/dask-worker/environment.yaml | 9 +++ docker/dask-worker/postBuild | 26 +++++++ docker/jupyterhub/environment.yaml | 17 +++++ docker/jupyterhub/postBuild | 5 ++ docker/jupyterlab/apt.txt | 50 +++++++++++++ docker/jupyterlab/environment.yaml | 69 ++++++++++++++++++ docker/jupyterlab/postBuild | 15 ++++ docker/nebari-workflow-controller/apt.txt | 26 +++++++ .../environment.yaml | 8 +++ docker/scripts/fix-permissions | 15 ++++ docker/scripts/install-apt-minimal.sh | 8 +++ docker/scripts/install-apt.sh | 15 ++++ docker/scripts/install-code-server.sh | 26 +++++++ docker/scripts/install-conda-environment.sh | 61 ++++++++++++++++ docker/scripts/install-conda.sh | 48 +++++++++++++ docker/scripts/install-gitlfs.sh | 20 ++++++ 20 files changed, 631 insertions(+) create mode 100644 docker/Dockerfile.dask-worker create mode 100644 docker/Dockerfile.jupyterhub create mode 100644 docker/Dockerfile.jupyterlab create mode 100644 docker/Dockerfile.workflow-controller create mode 100644 docker/dask-worker/environment.yaml create mode 100644 docker/dask-worker/postBuild create mode 100644 docker/jupyterhub/environment.yaml create mode 100644 docker/jupyterhub/postBuild create mode 100644 docker/jupyterlab/apt.txt create mode 100644 docker/jupyterlab/environment.yaml create mode 100644 docker/jupyterlab/postBuild create mode 100644 docker/nebari-workflow-controller/apt.txt create mode 100644 docker/nebari-workflow-controller/environment.yaml create mode 100644 docker/scripts/fix-permissions create mode 100644 docker/scripts/install-apt-minimal.sh create mode 100644 docker/scripts/install-apt.sh create mode 100644 docker/scripts/install-code-server.sh create mode 100644 docker/scripts/install-conda-environment.sh create mode 100644 docker/scripts/install-conda.sh create mode 100644 docker/scripts/install-gitlfs.sh diff --git a/docker/Dockerfile.dask-worker b/docker/Dockerfile.dask-worker new file mode 100644 index 000000000..693ecffa6 --- /dev/null +++ b/docker/Dockerfile.dask-worker @@ -0,0 +1,41 @@ +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. +# Usage: +# ------ +# +# To make a local build of the container, from the root directory: +# docker build -f Dockerfile.dask-worker -t nebari-dask-worker:latest . + +ARG BASE_IMAGE=ubuntu:20.04 +FROM $BASE_IMAGE +LABEL MAINTAINER="Nebari development team" + +COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh +RUN /opt/scripts/install-apt-minimal.sh + +COPY scripts/fix-permissions /opt/scripts/fix-permissions + +ENV MAMBAFORGE_VERSION 4.13.0-1 +ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 +ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c +SHELL ["/bin/bash", "-c"] + +ENV PATH=/opt/conda/bin:${PATH}:/opt/scripts + +# ============== base install =============== +COPY scripts/install-conda.sh /opt/scripts/install-conda.sh + +RUN /opt/scripts/install-conda.sh + +# ========== dask-worker install =========== +COPY dask-worker/environment.yaml /opt/dask-worker/environment.yaml +COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh +RUN /opt/scripts/install-conda-environment.sh /opt/dask-worker/environment.yaml 'false' + +# ========== Setup GPU Paths ============ +ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64 +ENV NVIDIA_PATH=/usr/local/nvidia/bin +ENV PATH="$NVIDIA_PATH:$PATH" + +COPY dask-worker /opt/dask-worker +RUN /opt/dask-worker/postBuild diff --git a/docker/Dockerfile.jupyterhub b/docker/Dockerfile.jupyterhub new file mode 100644 index 000000000..d1c696f94 --- /dev/null +++ b/docker/Dockerfile.jupyterhub @@ -0,0 +1,41 @@ +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. +# Usage: +# ------ +# +# To make a local build of the container, from the root directory: +# docker build -f Dockerfile.jupyterhub -t nebari-jupyterhub:latest . + +FROM ubuntu:20.04 +LABEL MAINTAINER="Nebari development team" + +COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh +RUN /opt/scripts/install-apt-minimal.sh + +COPY scripts/fix-permissions /opt/scripts/fix-permissions + +ENV MAMBAFORGE_VERSION 4.13.0-1 +ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 +ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c +SHELL ["/bin/bash", "-c"] + +ENV PATH="/opt/conda/bin:$PATH:/opt/scripts" + +# ============== base install =============== +COPY scripts/install-conda.sh /opt/scripts/install-conda.sh +RUN /opt/scripts/install-conda.sh + +# ========== jupyterhub install =========== +COPY jupyterhub/environment.yaml /opt/jupyterhub/environment.yaml +COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh +RUN /opt/scripts/install-conda-environment.sh /opt/jupyterhub/environment.yaml 'false' + +COPY jupyterhub /opt/jupyterhub +RUN /opt/jupyterhub/postBuild + +WORKDIR /srv/jupyterhub + +# So we can actually write a db file here +RUN fix-permissions /srv/jupyterhub + +CMD ["jupyterhub", "--config", "/usr/local/etc/jupyterhub/jupyterhub_config.py"] diff --git a/docker/Dockerfile.jupyterlab b/docker/Dockerfile.jupyterlab new file mode 100644 index 000000000..06a2d9fcf --- /dev/null +++ b/docker/Dockerfile.jupyterlab @@ -0,0 +1,70 @@ +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. +# Usage: +# ------ +# +# To make a local build of the container, from the root directory: +# docker build -f Dockerfile.jupyterlab -t nebari-jupyterlab:latest . + +ARG BASE_IMAGE=ubuntu:20.04 +FROM $BASE_IMAGE +LABEL MAINTAINER="Nebari development team" + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +RUN chmod -R a-w ~ +COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh +RUN /opt/scripts/install-apt-minimal.sh + +COPY scripts/fix-permissions /opt/scripts/fix-permissions + +ENV MAMBAFORGE_VERSION 4.13.0-1 +ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 +ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c +SHELL ["/bin/bash", "-c"] +ENV CONDA_DIR=/opt/conda \ + DEFAULT_ENV=default +# Set timezone +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set PATH for Dockerfile so that conda works and some useful scripts are +# available. Any changes intended to propagate to runtime containers should be +# set in /etc/profile.d (see setup_shell_behavior.sh) +ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/bin:/opt/conda/bin:${PATH}:/opt/scripts + +# ============= base install =============== +# install conda +COPY scripts/install-conda.sh /opt/scripts/install-conda.sh +RUN echo "${SHELL}"; env; cat ~/.bashrc; cat ~/.profile ; /opt/scripts/install-conda.sh + +# ========== jupyterlab install ============ +COPY jupyterlab/apt.txt /opt/jupyterlab/apt.txt +COPY scripts/install-apt.sh /opt/scripts/install-apt.sh +RUN /opt/scripts/install-apt.sh /opt/jupyterlab/apt.txt + +# Install extra packages (require custom package repository) +COPY scripts/install-gitlfs.sh /opt/scripts/install-gitlfs.sh +RUN /opt/scripts/install-gitlfs.sh + +ARG SKIP_CONDA_SOLVE=no +COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh +COPY jupyterlab/environment.yaml /opt/jupyterlab/environment.yaml +RUN \ + if [ "${SKIP_CONDA_SOLVE}" != "no" ];then \ + ENV_FILE=/opt/jupyterlab/conda-linux-64.lock ; \ + else \ + ENV_FILE=/opt/jupyterlab/environment.yaml ; \ + fi ; \ + /opt/scripts/install-conda-environment.sh "${ENV_FILE}" 'true' + +# ========== code-server install ============ +ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/share/code-server/bin:${PATH} +COPY scripts/install-code-server.sh /opt/scripts/install-code-server.sh + +COPY jupyterlab /opt/jupyterlab +RUN /opt/jupyterlab/postBuild + +# ========== Setup GPU Paths ============ +ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64 +ENV NVIDIA_PATH=/usr/local/nvidia/bin +ENV PATH="$NVIDIA_PATH:$PATH" diff --git a/docker/Dockerfile.workflow-controller b/docker/Dockerfile.workflow-controller new file mode 100644 index 000000000..968523936 --- /dev/null +++ b/docker/Dockerfile.workflow-controller @@ -0,0 +1,61 @@ +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. +# Usage: +# ------ +# +# To make a local build of the container, from the root directory: +# docker build -f Dockerfile.workflow-controller -t nebari-workflow-controller:latest . + +ARG BASE_IMAGE=ubuntu:20.04 +FROM $BASE_IMAGE +LABEL MAINTAINER="Nebari development team" + +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 +RUN chmod -R a-w ~ +COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh +RUN /opt/scripts/install-apt-minimal.sh + +COPY scripts/fix-permissions /opt/scripts/fix-permissions + +ENV MAMBAFORGE_VERSION 4.13.0-1 +ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 +ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c +SHELL ["/bin/bash", "-c"] +ENV CONDA_DIR=/opt/conda \ + DEFAULT_ENV=default +# Set timezone +ENV TZ=America/Chicago +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +# Set PATH for Dockerfile so that conda works and some useful scripts are +# available. Any changes intended to propagate to runtime containers should be +# set in /etc/profile.d (see setup_shell_behavior.sh) +ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/bin:/opt/conda/bin:${PATH}:/opt/scripts + +# ============= base install =============== +# install conda +COPY scripts/install-conda.sh /opt/scripts/install-conda.sh +RUN echo "${SHELL}"; env; cat ~/.bashrc; cat ~/.profile ; /opt/scripts/install-conda.sh + +# ========== nebari-workflow-controller install ============ +COPY scripts/install-apt.sh /opt/scripts/install-apt.sh +COPY nebari-workflow-controller/apt.txt /opt/nebari-workflow-controller/apt.txt +RUN /opt/scripts/install-apt.sh + +# uncomment to install dev dependencies +# RUN /opt/scripts/install-apt.sh /opt/nebari-workflow-controller/apt.txt + +ARG SKIP_CONDA_SOLVE=no +COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh +COPY nebari-workflow-controller/environment.yaml /opt/nebari-workflow-controller/environment.yaml +RUN \ + if [ "${SKIP_CONDA_SOLVE}" != "no" ];then \ + ENV_FILE=/opt/nebari-workflow-controller/conda-linux-64.lock ; \ + else \ + ENV_FILE=/opt/nebari-workflow-controller/environment.yaml ; \ + fi ; \ + /opt/scripts/install-conda-environment.sh "${ENV_FILE}" 'true' + +COPY nebari-workflow-controller /opt/nebari-workflow-controller + +CMD ["python", "-m", "nebari_workflow_controller"] \ No newline at end of file diff --git a/docker/dask-worker/environment.yaml b/docker/dask-worker/environment.yaml new file mode 100644 index 000000000..b41fc82d3 --- /dev/null +++ b/docker/dask-worker/environment.yaml @@ -0,0 +1,9 @@ +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +name: base +channels: + - conda-forge +dependencies: + # dask + - nebari-dask diff --git a/docker/dask-worker/postBuild b/docker/dask-worker/postBuild new file mode 100644 index 000000000..72cee969b --- /dev/null +++ b/docker/dask-worker/postBuild @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +set -euo pipefail + +# A workaround that allows a command to run in a +# specific conda environment +cat </opt/conda-run-worker +#!/bin/bash +set -xe + +source activate \$CONDA_ENVIRONMENT +dask-worker "\$@" +EOF + +cat </opt/conda-run-scheduler +#!/bin/bash +set -xe + +source activate \$CONDA_ENVIRONMENT +dask-scheduler "\$@" +EOF + +chmod 755 /opt/conda-run-worker +chmod 755 /opt/conda-run-scheduler diff --git a/docker/jupyterhub/environment.yaml b/docker/jupyterhub/environment.yaml new file mode 100644 index 000000000..df45cc4e6 --- /dev/null +++ b/docker/jupyterhub/environment.yaml @@ -0,0 +1,17 @@ +name: base +channels: + - conda-forge +dependencies: + - pip==21.1.2 + - jupyterhub==5.1.0 + - jupyterhub-kubespawner==4.2.0 + - oauthenticator==16.3.0 + - escapism==1.0.1 + - python-kubernetes + - kubernetes_asyncio==29.0.0 + - jupyterhub-idle-culler==1.2.1 + - sqlalchemy==1.4.46 + - pip: + - nebari-jupyterhub-theme==2024.7.1 + - python-keycloak==0.26.1 + - jhub-apps==2024.12.1 diff --git a/docker/jupyterhub/postBuild b/docker/jupyterhub/postBuild new file mode 100644 index 000000000..6b7e4da13 --- /dev/null +++ b/docker/jupyterhub/postBuild @@ -0,0 +1,5 @@ +#!/usr/bin/env bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +set -euo pipefail diff --git a/docker/jupyterlab/apt.txt b/docker/jupyterlab/apt.txt new file mode 100644 index 000000000..8546dfbd8 --- /dev/null +++ b/docker/jupyterlab/apt.txt @@ -0,0 +1,50 @@ +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +locales + +# assign uid/gid names +libnss-wrapper + +# utilities +wget +curl +htop +tree +zip +unzip + +# development utilities +git +openssh-client +tmux +xvfb +zsh + +# editors +nano +vim +emacs +neovim + +# conda prerequisites for GUI packages +# See https://docs.anaconda.com/anaconda/install/linux/ +libgl1-mesa-glx +libegl1-mesa +libxrandr2 +libxss1 +libxcursor1 +libxcomposite1 +libasound2 +libxi6 +libxtst6 +libfontconfig1 +libxrender1 +libosmesa6 + +# gpg +gnupg +pinentry-curses + +# extras +git-lfs diff --git a/docker/jupyterlab/environment.yaml b/docker/jupyterlab/environment.yaml new file mode 100644 index 000000000..73f835b83 --- /dev/null +++ b/docker/jupyterlab/environment.yaml @@ -0,0 +1,69 @@ +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +name: default +channels: + - conda-forge +dependencies: + # general + - pip + + # jupyterhub/jupyterlab + - nb_conda_kernels + - ipython > 7 + - jupyter-server-proxy >=4.4.0 + - "jupyter_server>=2.13.0" + - jupyterlab==4.2.5 + - jupyter_client + - jupyter_console + - jupyterhub==5.1.0 + - nbconvert + - nbval + + # jupyterhub extension + + # jupyterlab extensions + - dask_labextension >= 5.3.0 + - jupyterlab-git >=0.30.0 + - sidecar >=0.5.0 + - ipywidgets >= 8.0.0 + - ipyleaflet >=0.13.5 + - pyviz_comms >=3.0.3 + - jupyter-resource-usage >=0.6.0 + - nbgitpuller + - jupyterlab_code_formatter + - jupyterlab-spellchecker >= 0.7.3 + - jupyterlab-pioneer + - jupyter-ai + - jupyterlab-favorites >=3.2.1 + - jupyter-scheduler >=2.5.2,<2.6.0 + + # viz tools + - param + - python-graphviz + - plotly >=5.0 + - ipympl + - bokeh >=3.5.2 + + # testing, docs, linting + - pytest + - hypothesis + - flake8 + - sphinx + - pytest-cov + - black + - isort + - importnb + - git-lfs + + - pip: + # vscode jupyterlab launcher + - git+https://github.com/betatim/vscode-binder + - jupyterlab_nvdashboard==0.11.0 + - argo-jupyter-scheduler==2024.6.1 + - jhub-apps==2024.12.1 + - jupyterlab-nebari-mode==0.3.0 + - jupyterlab-conda-store==2024.11.1 + - jupyterlab-launchpad==1.0.3 + - jupyterlab-gallery==0.6.3 + - jupyterlab-jhub-apps==0.3.1 diff --git a/docker/jupyterlab/postBuild b/docker/jupyterlab/postBuild new file mode 100644 index 000000000..67fb47915 --- /dev/null +++ b/docker/jupyterlab/postBuild @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +set -euo pipefail +set -x + +# install code-server extension +sh /opt/scripts/install-code-server.sh "/opt/conda/envs/${DEFAULT_ENV}/share" + +# if DEFAULT_ENV is unset ${DEFAULT_ENV+x} expands to nothing otherwise +# it substitutes the string x. This allows us to check if the variable +# is set without triggering an unbound variable error +if [[ -z "${DEFAULT_ENV+x}" ]]; then + fix-permissions /opt/conda/bin +else + fix-permissions "/opt/conda/envs/${DEFAULT_ENV}" +fi diff --git a/docker/nebari-workflow-controller/apt.txt b/docker/nebari-workflow-controller/apt.txt new file mode 100644 index 000000000..05d484cea --- /dev/null +++ b/docker/nebari-workflow-controller/apt.txt @@ -0,0 +1,26 @@ +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +locales + +# assign uid/gid names +libnss-wrapper + +# utilities +wget +curl +htop +tree +zip +unzip + +# development utilities +git +openssh-client +tmux +xvfb + +# editors +nano +vim +emacs \ No newline at end of file diff --git a/docker/nebari-workflow-controller/environment.yaml b/docker/nebari-workflow-controller/environment.yaml new file mode 100644 index 000000000..62eef8bb3 --- /dev/null +++ b/docker/nebari-workflow-controller/environment.yaml @@ -0,0 +1,8 @@ +name: default +channels: + - conda-forge +dependencies: + - python=3.10 + - pip + - pip: + - nebari-workflow-controller==2023.7.1 diff --git a/docker/scripts/fix-permissions b/docker/scripts/fix-permissions new file mode 100644 index 000000000..8e9926c02 --- /dev/null +++ b/docker/scripts/fix-permissions @@ -0,0 +1,15 @@ +#!/bin/bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +# uses find to avoid touching files that already have the right permissions +# right permissions are: +# world rX, we have no guarantees of uids or gids upon +# deployment so we want files accessible to all. + +set -e +for d in "$@"; do + find "$d" \ + ! -perm -o+rX \ + -exec chmod o+rX {} \; +done diff --git a/docker/scripts/install-apt-minimal.sh b/docker/scripts/install-apt-minimal.sh new file mode 100644 index 000000000..297177dca --- /dev/null +++ b/docker/scripts/install-apt-minimal.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +apt-get update --fix-missing && + apt-get install -y wget bzip2 ca-certificates curl git && + apt-get clean && + rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* diff --git a/docker/scripts/install-apt.sh b/docker/scripts/install-apt.sh new file mode 100644 index 000000000..6c142f46b --- /dev/null +++ b/docker/scripts/install-apt.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +set -xe + +# Assumes apt packages installs packages in "$1" argument + +# ====== install apt packages ======== +apt-get update +apt-get install -y --no-install-recommends $(grep -vE "^\s*#" $1 | tr "\n" " ") + +# ========== cleanup apt ============= +apt-get clean +rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* diff --git a/docker/scripts/install-code-server.sh b/docker/scripts/install-code-server.sh new file mode 100644 index 000000000..9c5700c65 --- /dev/null +++ b/docker/scripts/install-code-server.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +set -xe +DEFAULT_PREFIX="${1}" +shift # path to environment yaml or lock file +CODE_SERVER_VERSION=4.23.1 + +mkdir -p ${DEFAULT_PREFIX}/code-server +cd ${DEFAULT_PREFIX}/code-server + +# Fetch the snapshot of https://code-server.dev/install.sh as of the time of writing +wget --quiet https://raw.githubusercontent.com/coder/code-server/v4.23.1/install.sh +expected_sum=ef0324043bc7493989764315e22bbc85c38c4e895549538b7e701948b64495e6 + +if [[ ! $(sha256sum install.sh) == "${expected_sum} install.sh" ]]; then + echo Unexpected hash from code-server install script + exit 1 +fi + +mkdir /opt/tmpdir +sh ./install.sh --method standalone --prefix /opt/tmpdir --version ${CODE_SERVER_VERSION} + +mv /opt/tmpdir/lib/code-server-${CODE_SERVER_VERSION}/* ${DEFAULT_PREFIX}/code-server +rm -rf /opt/tmpdir diff --git a/docker/scripts/install-conda-environment.sh b/docker/scripts/install-conda-environment.sh new file mode 100644 index 000000000..03ef7cc04 --- /dev/null +++ b/docker/scripts/install-conda-environment.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +set -xe +ENV_FILE="${1}" +shift # path to environment yaml or lock file +NEW_ENV="${1}" +shift # true or false indicating whether env update should occur + +# Capture last optional arg or set a ENV_NAME. This can be changed but be +# careful... setting the path for both the dockerfile and runtime container +# can be tricky +if [[ -z "${1+x}" ]] || [[ "${1}" == "" ]]; then + ENV_NAME=default +else + ENV_NAME="${1}" + shift +fi + +# Set a default value for skipping the conda solve (using a lock file). +: ${SKIP_CONDA_SOLVE:=no} + +# ==== install conda dependencies ==== + +if ! ${NEW_ENV}; then + if [[ $(basename "${ENV_FILE}") =~ "*lock*" ]]; then + echo "${ENV_FILE} should not be a lock file as this is not supported when \ + only updating the conda environment. Consider setting NEW_ENV to yes." + exit 1 + fi + echo Installing into current conda environment + mamba env update -f "${ENV_FILE}" + +# Env not being updated... create one now: +elif [[ "${SKIP_CONDA_SOLVE}" == "no" ]]; then + mamba env create --prefix=/opt/conda/envs/${ENV_NAME} -f "${ENV_FILE}" +elif [[ "${SKIP_CONDA_SOLVE}" == "yes" ]]; then + mamba create --prefix=/opt/conda/envs/${ENV_NAME} --file "${ENV_FILE}" + + # This needs to be set using the ENV directive in the docker file + PATH="/opt/conda/envs/${ENV_NAME}/bin:${PATH}" + # For now install pip section manually. We could consider using pip-tools... + # See https://github.com/conda-incubator/conda-lock/issues/4 + pip install https://github.com/dirkcgrunwald/jupyter_codeserver_proxy-/archive/5596bc9c2fbd566180545fa242c659663755a427.tar.gz +else + echo "SKIP_CONDA_SOLVE should be yes or no instead got: '${SKIP_CONDA_SOLVE}'" + exit 1 +fi + +# ========= list dependencies ======== +/opt/conda/bin/conda list + +# ========== cleanup conda =========== +/opt/conda/bin/mamba clean -afy +# remove unnecissary files (statis, js.maps) +find /opt/conda/ -follow -type f -name '*.a' -delete +find /opt/conda/ -follow -type f -name '*.js.map' -delete + +# Fix permissions +fix-permissions "/opt/conda/envs/${ENV_NAME}" || fix-permissions /opt/conda/bin diff --git a/docker/scripts/install-conda.sh b/docker/scripts/install-conda.sh new file mode 100644 index 000000000..43f018619 --- /dev/null +++ b/docker/scripts/install-conda.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +set -xe + +# Requires environment MAMBAFORGE_SHA256, MINIFORGE_VERSION, and DEFAULT_ENV +arch=$(uname -i) +wget --quiet -O mambaforge.sh https://github.com/conda-forge/miniforge/releases/download/$MAMBAFORGE_VERSION/Mambaforge-Linux-$arch.sh + +if [[ $arch == "aarch64" ]]; then + echo "${MAMBAFORGE_AARCH64_SHA256} mambaforge.sh" >mambaforge.checksum +elif [[ $arch == "x86_64" ]]; then + echo "${MAMBAFORGE_X86_64_SHA256} mambaforge.sh" >mambaforge.checksum +else + echo "Unsupported architecture: $arch" + exit 1 +fi + +echo $(sha256sum -c mambaforge.checksum) + +if [ $(sha256sum -c mambaforge.checksum | awk '{print $2}') != "OK" ]; then + echo Error when testing checksum + exit 1 +fi + +# Install Mamba and clean-up +sh ./mambaforge.sh -b -p /opt/conda +rm mambaforge.sh mambaforge.checksum + +mamba --version +mamba clean -afy + +ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh + +mkdir -p /etc/conda +cat </etc/conda/condarc +always_yes: true +changeps1: false +auto_update_conda: false +aggressive_update_packages: [] +envs_dirs: + - /home/conda/environments +EOF + +# Fix permissions in accordance with jupyter stack permissions +# model +fix-permissions /opt/conda /etc/conda /etc/profile.d diff --git a/docker/scripts/install-gitlfs.sh b/docker/scripts/install-gitlfs.sh new file mode 100644 index 000000000..47dc9177f --- /dev/null +++ b/docker/scripts/install-gitlfs.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Copyright (c) Nebari Development Team. +# Distributed under the terms of the Modified BSD License. + +set -xe + +# Adding the packagecloud repository for git-lfs installation +wget --quiet -O script.deb.sh https://packagecloud.io/install/repositories/github/git-lfs/script.deb.sh +expected_sum=8c4d07257b8fb6d612b6085f68ad33c34567b00d0e4b29ed784b2a85380f727b + +if [[ ! $(sha256sum script.deb.sh) == "${expected_sum} script.deb.sh" ]]; then + echo Unexpected hash from git-lfs install script + exit 1 +fi + +# Install packagecloud's repository signing key and add repository to apt +bash ./script.deb.sh + +# Install git-lfs +apt-get install -y --no-install-recommends git-lfs From 48bcb2d045d880beaf514d7bffeca4d51c898200 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Sat, 1 Feb 2025 09:40:41 +0530 Subject: [PATCH 02/29] workflow updated for docker setup --- .github/workflows/build-push-docker.yaml | 122 +++++++++++++++++++++++ .github/workflows/test-images.yaml | 68 +++++++++++++ 2 files changed, 190 insertions(+) create mode 100644 .github/workflows/build-push-docker.yaml create mode 100644 .github/workflows/test-images.yaml diff --git a/.github/workflows/build-push-docker.yaml b/.github/workflows/build-push-docker.yaml new file mode 100644 index 000000000..c943e96e1 --- /dev/null +++ b/.github/workflows/build-push-docker.yaml @@ -0,0 +1,122 @@ +# Build and push images to: +# GitHub Container Registry (ghcr.io) +# Red Hat Container Registry (quay.io) +name: "Build Docker Images" + +on: + workflow_dispatch: null + push: + branches: + - "*" + paths: + - "docker/Dockerfile.*" + - "docker/dask-worker/*" + - "docker/jupyterhub/*" + - "docker/jupyterlab/*" + - "docker/nebari-workflow-controller/*" + + - "docker/scripts/*" + + - ".github/workflows/build-push-docker.yaml" + tags: + - "*" + +env: + DOCKER_ORG: nebari + GPU_BASE_IMAGE: nvidia/cuda:12.2.2-base-ubuntu20.04 + GPU_IMAGE_SUFFIX: gpu + BASE_IMAGE: ubuntu:20.04 + +permissions: + contents: read + packages: write + id-token: write + security-events: write + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-images: + name: "Build Docker Images" + runs-on: ubuntu-latest + strategy: + matrix: + dockerfile: + - jupyterlab + - jupyterhub + - dask-worker + - workflow-controller + platform: + - gpu + - cpu + exclude: + # excludes JupyterHub/GPU, Workflow Controller/GPU + - dockerfile: jupyterhub + platform: gpu + - dockerfile: workflow-controller + platform: gpu + + steps: + - name: "Checkout Repository ๐Ÿ›Ž๏ธ" + uses: actions/checkout@v3 + + - name: "Set up Docker Buildx ๐Ÿ› ๏ธ" + uses: docker/setup-buildx-action@v2 + + - name: "Login to GitHub Container Registry ๐Ÿ”" + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.BOT_GHCR_TOKEN }} + + - name: "Login to Quay Container Registry ๐Ÿ”" + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_TOKEN }} + + - name: "Set BASE_IMAGE and Image Suffix ๐Ÿ“ท" + if: ${{ matrix.platform == 'gpu' }} + run: | + echo "GPU Platform Matrix" + echo "BASE_IMAGE=$GPU_BASE_IMAGE" >> $GITHUB_ENV + echo "IMAGE_SUFFIX=-$GPU_IMAGE_SUFFIX" >> $GITHUB_ENV + + - name: "Generate Docker images tags ๐Ÿท๏ธ" + id: meta + uses: docker/metadata-action@v4 + with: + images: | + "quay.io/${{ env.DOCKER_ORG }}/nebari-${{ matrix.dockerfile }}${{ env.IMAGE_SUFFIX }}" + "ghcr.io/${{ github.repository_owner }}/nebari-${{ matrix.dockerfile }}${{ env.IMAGE_SUFFIX }}" + tags: | + # branch event -> e.g. `main-f0f6994-20221001` + type=ref, event=branch, suffix=-{{sha}}-{{date 'YYYYMMDD'}} + # needed for integration tests + type=ref, event=branch + # on tag push -> e.g. `2022.10.1` + type=ref, event=tag + + - name: "Inspect image dir tree ๐Ÿ”" + run: | + sudo apt-get install tree + tree . + + - name: "Build docker images ๐Ÿณ" + uses: docker/build-push-action@v3 + with: + context: . + file: "Dockerfile.${{ matrix.dockerfile }}" + tags: ${{ steps.meta.outputs.tags }} + push: ${{ github.event_name != 'pull_request' }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BASE_IMAGE=${{ env.BASE_IMAGE }} + platforms: linux/amd64,linux/arm64 \ No newline at end of file diff --git a/.github/workflows/test-images.yaml b/.github/workflows/test-images.yaml new file mode 100644 index 000000000..30928496c --- /dev/null +++ b/.github/workflows/test-images.yaml @@ -0,0 +1,68 @@ +name: Test Docker images + +on: + pull_request: + paths: + - "docker/Dockerfile.*" + + - "docker/dask-worker/*" + - "docker/jupyterhub/*" + - "docker/jupyterlab/*" + + - "docker/scripts/*" + + - ".github/workflows/build-push-docker.yaml" + - ".github/workflows/test-images.yaml" + +env: + DOCKER_ORG: nebari + GITHUB_SHA: ${{ github.sha }} + GPU_BASE_IMAGE: nvidia/cuda:12.2.2-base-ubuntu20.04 + GPU_IMAGE_SUFFIX: gpu + BASE_IMAGE: ubuntu:20.04 + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-test-images: + runs-on: ubuntu-latest + strategy: + matrix: + dockerfile: + - jupyterlab + - jupyterhub + - dask-worker + platform: + - gpu + - cpu + exclude: + # excludes JupyterHub/GPU + - dockerfile: jupyterhub + platform: gpu + steps: + - name: Checkout Repository ๐Ÿ›Ž + uses: actions/checkout@v3 + + - name: Lint Dockerfiles ๐Ÿ” + uses: jbergstroem/hadolint-gh-action@v1 + with: + dockerfile: Dockerfile.${{ matrix.dockerfile }} + output_format: tty + error_level: 0 + + - name: "Set BASE_IMAGE and Image Suffix ๐Ÿ“ท" + if: ${{ matrix.platform == 'gpu' }} + run: | + echo "GPU Platform Matrix" + echo "BASE_IMAGE=$GPU_BASE_IMAGE" >> $GITHUB_ENV + echo "IMAGE_SUFFIX=-$GPU_IMAGE_SUFFIX" >> $GITHUB_ENV + + - name: Build Image ๐Ÿ›  + run: | + docker build -t ${DOCKER_ORG}/${{ matrix.dockerfile }}${{ env.IMAGE_SUFFIX }}:${{ env.GITHUB_SHA }} \ + --build-arg BASE_IMAGE=$BASE_IMAGE \ + -f Dockerfile.${{ matrix.dockerfile }} . From d5c0387f18f02e76abd629c08854b1f950bb1274 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Sat, 1 Feb 2025 09:51:05 +0530 Subject: [PATCH 03/29] added documention and rfd from docker repo --- .github/ISSUE_TEMPLATE/RFD.md | 52 ++++++++++++++++++++++ .github/ISSUE_TEMPLATE/documentation.yml | 56 ++++++++++++++++++++++++ 2 files changed, 108 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/RFD.md create mode 100644 .github/ISSUE_TEMPLATE/documentation.yml diff --git a/.github/ISSUE_TEMPLATE/RFD.md b/.github/ISSUE_TEMPLATE/RFD.md new file mode 100644 index 000000000..ab29a17d4 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/RFD.md @@ -0,0 +1,52 @@ +--- +name: "Request for Discussion (RFD) ๐Ÿ’ฌ" +about: Open discussion about a feature or design project in Nebari. +labels: + - "type: RFD" +title: "RFD - Title" +--- + + + + + + +| Status | Draft ๐Ÿšง / Open for comments ๐Ÿ’ฌ/ Accepted โœ… /Implemented ๐Ÿš€/ Obsolete ๐Ÿ—ƒ | +| ----------------- | ------------------------------------------------------------------------ | +| Author(s) | GitHub handle | +| Date Created | dd-MM-YYY | +| Date Last updated | dd-MM-YYY | +| Decision deadline | dd-MM-YYY | + +# Title + +## Summary + + + +## User benefit + + + +## Design Proposal + + + +### Alternatives or approaches considered (if any) + + + +### Best practices + + + +### User impact + + + +## Unresolved questions + + diff --git a/.github/ISSUE_TEMPLATE/documentation.yml b/.github/ISSUE_TEMPLATE/documentation.yml new file mode 100644 index 000000000..cf300669c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation.yml @@ -0,0 +1,56 @@ +name: "Documentation ๐Ÿ“–" +description: Did you find an error in our documentation? Report your findings here. +title: "[DOC] - " +labels: ["area: documentation ๐Ÿ“–"] + +body: + - type: markdown + attributes: + value: | + # Welcome ๐Ÿ‘‹ + + Thanks for using Nebari and taking some time to contribute to this project. + + Please fill out each section below. This info allows Nebari maintainers to diagnose (and fix!) your issue as + quickly as possible. + Before submitting a bug, please make sure the issue hasn't been already addressed by searching through + [the past issues](https://github.com/nebari-dev/nebari-docs/issues). + + Useful links: + + - Documentation: https://www.nebari.dev + - Contribution guidelines: https://www.nebari.dev/community/ + + - type: checkboxes + attributes: + label: Preliminary Checks + description: Please make sure that you verify each checkbox and follow the instructions for them. + options: + - label: "This issue is not a question, feature request, RFC, or anything other than a bug report. Please post those things in GitHub Discussions: https://github.com/nebari-dev/nebari/discussions" + required: true + - type: textarea + validations: + required: true + attributes: + label: Summary + description: | + What problem(s) did you run into that caused you to request a fix to the documentation or additional + documentation? What questions do you think we should answer? + + - type: textarea + validations: + required: true + attributes: + label: Steps to Resolve this Issue + description: | + How can the problem be solved? Are there any additional steps required? Do any other pages need to be updated? + value: | + 1. + 2. + 3. + ... + + - type: markdown + attributes: + value: > + Thanks for contributing ๐ŸŽ‰! From 6bda9fdc7d2848d8d05db2dcca0557f27a70d8b0 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Sat, 1 Feb 2025 09:53:25 +0530 Subject: [PATCH 04/29] Update .pre-commit-config.yaml from docker repo --- .pre-commit-config.yaml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index d566b6f38..c752f6cb5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -82,3 +82,28 @@ repos: - id: terraform_fmt args: - --args=-write=true + + # Autoformat: markdown, yaml to ensure that it doesn't need to be updated in other repos + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.6.1 + hooks: + - id: prettier + + # Misc... + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.0.1 + # ref: https://github.com/pre-commit/pre-commit-hooks#hooks-available + hooks: + # Autoformat: Makes sure files end in a newline and only a newline. + - id: end-of-file-fixer + + # Trims trailing whitespace. + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + + # Lint: Check for files with names that would conflict on a + # case-insensitive filesystem like MacOS HFS+ or Windows FAT. + - id: check-case-conflict + + # Lint: Checks that non-binary executables have a proper shebang. + - id: check-executables-have-shebangs From a94a2c29d456e93b1b1e734c6f4a5b35e4009f22 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 7 Feb 2025 22:14:37 +0530 Subject: [PATCH 05/29] Added Readme.md from the docker repository --- docker/README.md | 121 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 121 insertions(+) create mode 100644 docker/README.md diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 000000000..cee5c7f28 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,121 @@ +<p align="center"> +<picture> + <source media="(prefers-color-scheme: light)" srcset="https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup.svg"> + <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg"> + <img alt="Nebari logo mark - text will be black in light color mode and white in dark color mode." src="https://raw.githubusercontent.com/nebari-dev/nebari-design/main/logo-mark/horizontal/Nebari-Logo-Horizontal-Lockup-White-text.svg" width="50%"/> +</picture> +</p> + +--- + +# Nebari base Docker images + +| Information | Links | +| :---------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Project | [![License - BSD3 License badge](https://img.shields.io/badge/License-BSD%203--Clause-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)](https://opensource.org/licenses/BSD-3-Clause) [![Nebari documentation badge - nebari.dev](https://img.shields.io/badge/%F0%9F%93%96%20Read-the%20docs-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)][nebari-docs] | +| Community | [![GH discussions badge](https://img.shields.io/badge/%F0%9F%92%AC%20-Participate%20in%20discussions-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)][nebari-discussions] [![Open a GH issue badge](https://img.shields.io/badge/%F0%9F%93%9D%20Open-an%20issue-gray.svg?colorA=2D2A56&colorB=5936D9&style=flat.svg)][nebari-docker-issues] | +| CI | ![Build Docker Images - GitHub action status badge](https://github.com/nebari-dev/nebari-docker-images/actions/workflows/build-push-docker.yaml/badge.svg) | + +- [Nebari base Docker images](#nebari-base-docker-images) + - [Getting started โšก๏ธ](#getting-started-๏ธ) + - [Prerequisites ๐Ÿ’ป](#prerequisites-) + - [Building the Docker images ๐Ÿ› ](#building-the-docker-images-) + - [Pre-commit hooks ๐Ÿงน](#pre-commit-hooks-) + - [Reporting an issue ๐Ÿ“](#reporting-an-issue-) + - [Contributions ๐Ÿค](#contributions-) + - [License ๐Ÿ“„](#license-) + +This repository contains the source code for Docker (container) images used by the [Nebari platform][nebari-docs]. It also contains an automated means of building and pushing these images to public container registries through [GitHub actions][nebari-docker-actions]. Currently, these images are built and pushed to the following registries: + +**GitHub Container Registry (ghcr.io)** + +- [`nebari-jupyterlab`](https://github.com/orgs/nebari-dev/packages/container/package/nebari-jupyterlab) +- [`nebari-jupyterlab-gpu`](https://github.com/orgs/nebari-dev/packages/container/package/nebari-jupyterlab-gpu) +- [`nebari-jupyterhub`](https://github.com/orgs/nebari-dev/packages/container/package/nebari-jupyterhub) +- [`nebari-dask-worker`](https://github.com/orgs/nebari-dev/packages/container/package/nebari-dask-worker) +- [`nebari-dask-worker-gpu`](https://github.com/orgs/nebari-dev/packages/container/package/nebari-dask-worker-gpu) + +**Quay Container Registry (quay.io)** + +- [`nebari-jupyterlab`](https://quay.io/repository/nebari/nebari-jupyterlab) +- [`nebari-jupyterlab-gpu`](https://quay.io/repository/nebari/nebari-jupyterlab-gpu) +- [`nebari-jupyterhub`](https://quay.io/repository/nebari/nebari-jupyterhub) +- [`nebari-dask-worker`](https://quay.io/repository/nebari/nebari-dask-worker) +- [`nebari-dask-worker-gpu`](https://quay.io/repository/nebari/nebari-dask-worker-gpu) + +## Getting started โšก๏ธ + +Whether you want to contribute to this project or whether you wish use these images, to get started, fork this repo and then clone the forked repo onto your local machine. + +### Prerequisites ๐Ÿ’ป + +- [`docker`](https://docs.docker.com/get-docker/), make sure to read the [Docker official documentation on how to install Docker on your machine](https://docs.docker.com/get-docker/). +- [pre-commit](https://pre-commit.com/), which can be installed with: + + ```bash + pip install pre-commit + # or using conda + conda install -c conda-forge pre-commit + ``` + +### Building the Docker images ๐Ÿ›  + +From the repository's root folder, you can build these images locally by running the listed commands on your terminal. + +- For [JupyterLab](Dockerfile.jupyterlab): + + ```shell + docker build -f Dockerfile.jupyterlab \ + -t nebari-jupyterlab:latest . + ``` + +- For [JupyterHub](Dockerfile.jupyterhub): + + ```shell + docker build -f Dockerfile.jupyterhub \ + -t nebari-jupyterhub:latest . + ``` + +- For [Dask-Worker](Dockerfile.dask-worker): + + ```shell + docker build -f Dockerfile.dask-gateway \ + -t nebari-dask-gateway:latest . + ``` + +> **NOTE** +> It is extremely important to pin specific packages `dask-gateway` and `distributed` as they need to run the same version for the `dask-workers` to work as expected. + +### Pre-commit hooks ๐Ÿงน + +This repository uses the `prettier` pre-commit hook to standardize our YAML and markdown structure. +To install and run it, use these commands from the repository root: + +```bash +# install the pre-commit hooks +pre-commit install + +# run the pre-commit hooks +pre-commit run --all-files +``` + +## Reporting an issue ๐Ÿ“ + +If you encounter an issue or want to make suggestions on how we can make this project better, feel free to [open an issue on this repository's issue tracker](https://github.com/nebari-dev/nebari-docker-images/issues/new/choose). + +## Contributions ๐Ÿค + +Thinking about contributing to this repository or any other in the Nebari org? Check out our +[Contribution Guidelines](https://nebari.dev/community). + +## License ๐Ÿ“„ + +[Nebari is BSD3 licensed](LICENSE). + +<!-- Links --> + +[nebari-docker-repo]: https://github.com/nebari-dev/nebari-docker-images +[nebari-docker-issues]: https://github.com/nebari-dev/nebari-docker-images/issues/new/choose +[nebari-docker-actions]: https://github.com/nebari-dev/nebari-docker-images/actions +[nebari-discussions]: https://github.com/orgs/nebari-dev/discussions +[nebari-docs]: https://nebari.dev From 99fe60aa047666b54aa614a7466d10a881444ca2 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Tue, 11 Feb 2025 06:59:00 +0530 Subject: [PATCH 06/29] Update install-conda-environment.sh --- docker/scripts/install-conda-environment.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/scripts/install-conda-environment.sh b/docker/scripts/install-conda-environment.sh index 03ef7cc04..5ad070dfc 100644 --- a/docker/scripts/install-conda-environment.sh +++ b/docker/scripts/install-conda-environment.sh @@ -53,7 +53,7 @@ fi # ========== cleanup conda =========== /opt/conda/bin/mamba clean -afy -# remove unnecissary files (statis, js.maps) +# remove unnecissary files (status, js.maps) find /opt/conda/ -follow -type f -name '*.a' -delete find /opt/conda/ -follow -type f -name '*.js.map' -delete From d23853bd5887eb83eda9dbcab5ff4cf423196c28 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Wed, 23 Apr 2025 09:06:10 +0530 Subject: [PATCH 07/29] Removed .github/workflows/build-push-docker.yaml --- .github/workflows/build-push-docker.yaml | 122 ----------------------- 1 file changed, 122 deletions(-) delete mode 100644 .github/workflows/build-push-docker.yaml diff --git a/.github/workflows/build-push-docker.yaml b/.github/workflows/build-push-docker.yaml deleted file mode 100644 index c943e96e1..000000000 --- a/.github/workflows/build-push-docker.yaml +++ /dev/null @@ -1,122 +0,0 @@ -# Build and push images to: -# GitHub Container Registry (ghcr.io) -# Red Hat Container Registry (quay.io) -name: "Build Docker Images" - -on: - workflow_dispatch: null - push: - branches: - - "*" - paths: - - "docker/Dockerfile.*" - - "docker/dask-worker/*" - - "docker/jupyterhub/*" - - "docker/jupyterlab/*" - - "docker/nebari-workflow-controller/*" - - - "docker/scripts/*" - - - ".github/workflows/build-push-docker.yaml" - tags: - - "*" - -env: - DOCKER_ORG: nebari - GPU_BASE_IMAGE: nvidia/cuda:12.2.2-base-ubuntu20.04 - GPU_IMAGE_SUFFIX: gpu - BASE_IMAGE: ubuntu:20.04 - -permissions: - contents: read - packages: write - id-token: write - security-events: write - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build-images: - name: "Build Docker Images" - runs-on: ubuntu-latest - strategy: - matrix: - dockerfile: - - jupyterlab - - jupyterhub - - dask-worker - - workflow-controller - platform: - - gpu - - cpu - exclude: - # excludes JupyterHub/GPU, Workflow Controller/GPU - - dockerfile: jupyterhub - platform: gpu - - dockerfile: workflow-controller - platform: gpu - - steps: - - name: "Checkout Repository ๐Ÿ›Ž๏ธ" - uses: actions/checkout@v3 - - - name: "Set up Docker Buildx ๐Ÿ› ๏ธ" - uses: docker/setup-buildx-action@v2 - - - name: "Login to GitHub Container Registry ๐Ÿ”" - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.repository_owner }} - password: ${{ secrets.BOT_GHCR_TOKEN }} - - - name: "Login to Quay Container Registry ๐Ÿ”" - uses: docker/login-action@v2 - with: - registry: quay.io - username: ${{ secrets.QUAY_USERNAME }} - password: ${{ secrets.QUAY_TOKEN }} - - - name: "Set BASE_IMAGE and Image Suffix ๐Ÿ“ท" - if: ${{ matrix.platform == 'gpu' }} - run: | - echo "GPU Platform Matrix" - echo "BASE_IMAGE=$GPU_BASE_IMAGE" >> $GITHUB_ENV - echo "IMAGE_SUFFIX=-$GPU_IMAGE_SUFFIX" >> $GITHUB_ENV - - - name: "Generate Docker images tags ๐Ÿท๏ธ" - id: meta - uses: docker/metadata-action@v4 - with: - images: | - "quay.io/${{ env.DOCKER_ORG }}/nebari-${{ matrix.dockerfile }}${{ env.IMAGE_SUFFIX }}" - "ghcr.io/${{ github.repository_owner }}/nebari-${{ matrix.dockerfile }}${{ env.IMAGE_SUFFIX }}" - tags: | - # branch event -> e.g. `main-f0f6994-20221001` - type=ref, event=branch, suffix=-{{sha}}-{{date 'YYYYMMDD'}} - # needed for integration tests - type=ref, event=branch - # on tag push -> e.g. `2022.10.1` - type=ref, event=tag - - - name: "Inspect image dir tree ๐Ÿ”" - run: | - sudo apt-get install tree - tree . - - - name: "Build docker images ๐Ÿณ" - uses: docker/build-push-action@v3 - with: - context: . - file: "Dockerfile.${{ matrix.dockerfile }}" - tags: ${{ steps.meta.outputs.tags }} - push: ${{ github.event_name != 'pull_request' }} - labels: ${{ steps.meta.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max - build-args: BASE_IMAGE=${{ env.BASE_IMAGE }} - platforms: linux/amd64,linux/arm64 \ No newline at end of file From 7ec2f0618057050f47bbe38c25dd93886f8f25b2 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Wed, 23 Apr 2025 09:06:36 +0530 Subject: [PATCH 08/29] Removed .github/workflows/test-images.yaml --- .github/workflows/test-images.yaml | 68 ------------------------------ 1 file changed, 68 deletions(-) delete mode 100644 .github/workflows/test-images.yaml diff --git a/.github/workflows/test-images.yaml b/.github/workflows/test-images.yaml deleted file mode 100644 index 30928496c..000000000 --- a/.github/workflows/test-images.yaml +++ /dev/null @@ -1,68 +0,0 @@ -name: Test Docker images - -on: - pull_request: - paths: - - "docker/Dockerfile.*" - - - "docker/dask-worker/*" - - "docker/jupyterhub/*" - - "docker/jupyterlab/*" - - - "docker/scripts/*" - - - ".github/workflows/build-push-docker.yaml" - - ".github/workflows/test-images.yaml" - -env: - DOCKER_ORG: nebari - GITHUB_SHA: ${{ github.sha }} - GPU_BASE_IMAGE: nvidia/cuda:12.2.2-base-ubuntu20.04 - GPU_IMAGE_SUFFIX: gpu - BASE_IMAGE: ubuntu:20.04 - -# https://docs.github.com/en/actions/using-jobs/using-concurrency -concurrency: - # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - build-test-images: - runs-on: ubuntu-latest - strategy: - matrix: - dockerfile: - - jupyterlab - - jupyterhub - - dask-worker - platform: - - gpu - - cpu - exclude: - # excludes JupyterHub/GPU - - dockerfile: jupyterhub - platform: gpu - steps: - - name: Checkout Repository ๐Ÿ›Ž - uses: actions/checkout@v3 - - - name: Lint Dockerfiles ๐Ÿ” - uses: jbergstroem/hadolint-gh-action@v1 - with: - dockerfile: Dockerfile.${{ matrix.dockerfile }} - output_format: tty - error_level: 0 - - - name: "Set BASE_IMAGE and Image Suffix ๐Ÿ“ท" - if: ${{ matrix.platform == 'gpu' }} - run: | - echo "GPU Platform Matrix" - echo "BASE_IMAGE=$GPU_BASE_IMAGE" >> $GITHUB_ENV - echo "IMAGE_SUFFIX=-$GPU_IMAGE_SUFFIX" >> $GITHUB_ENV - - - name: Build Image ๐Ÿ›  - run: | - docker build -t ${DOCKER_ORG}/${{ matrix.dockerfile }}${{ env.IMAGE_SUFFIX }}:${{ env.GITHUB_SHA }} \ - --build-arg BASE_IMAGE=$BASE_IMAGE \ - -f Dockerfile.${{ matrix.dockerfile }} . From e398daf2b4574e8f28178c7484e8d6ea336025f1 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 17:54:50 +0530 Subject: [PATCH 09/29] Update jupiterhub environment.yaml to 5.3.0 --- docker/jupyterhub/environment.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/jupyterhub/environment.yaml b/docker/jupyterhub/environment.yaml index df45cc4e6..d313d3e3a 100644 --- a/docker/jupyterhub/environment.yaml +++ b/docker/jupyterhub/environment.yaml @@ -3,8 +3,8 @@ channels: - conda-forge dependencies: - pip==21.1.2 - - jupyterhub==5.1.0 - - jupyterhub-kubespawner==4.2.0 + - jupyterhub==5.3.0 + - jupyterhub-kubespawner==6.2.0 - oauthenticator==16.3.0 - escapism==1.0.1 - python-kubernetes @@ -14,4 +14,4 @@ dependencies: - pip: - nebari-jupyterhub-theme==2024.7.1 - python-keycloak==0.26.1 - - jhub-apps==2024.12.1 + - jhub-apps==2025.2.1 From 740ecbeee9a2868b7d7811f291806a8cbb948386 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 17:56:23 +0530 Subject: [PATCH 10/29] Delete docker/jupyterlab/apt.txt --- docker/jupyterlab/apt.txt | 50 --------------------------------------- 1 file changed, 50 deletions(-) delete mode 100644 docker/jupyterlab/apt.txt diff --git a/docker/jupyterlab/apt.txt b/docker/jupyterlab/apt.txt deleted file mode 100644 index 8546dfbd8..000000000 --- a/docker/jupyterlab/apt.txt +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (c) Nebari Development Team. -# Distributed under the terms of the Modified BSD License. - -locales - -# assign uid/gid names -libnss-wrapper - -# utilities -wget -curl -htop -tree -zip -unzip - -# development utilities -git -openssh-client -tmux -xvfb -zsh - -# editors -nano -vim -emacs -neovim - -# conda prerequisites for GUI packages -# See https://docs.anaconda.com/anaconda/install/linux/ -libgl1-mesa-glx -libegl1-mesa -libxrandr2 -libxss1 -libxcursor1 -libxcomposite1 -libasound2 -libxi6 -libxtst6 -libfontconfig1 -libxrender1 -libosmesa6 - -# gpg -gnupg -pinentry-curses - -# extras -git-lfs From 8bd45118d29630d463c453a004ceb8ae677f9ecc Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 17:57:53 +0530 Subject: [PATCH 11/29] Upgrade JupyterHub to 5.3.0 --- docker/jupyterlab/environment.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docker/jupyterlab/environment.yaml b/docker/jupyterlab/environment.yaml index 73f835b83..ceac12c94 100644 --- a/docker/jupyterlab/environment.yaml +++ b/docker/jupyterlab/environment.yaml @@ -13,10 +13,10 @@ dependencies: - ipython > 7 - jupyter-server-proxy >=4.4.0 - "jupyter_server>=2.13.0" - - jupyterlab==4.2.5 + - jupyterlab==4.4.2 - jupyter_client - jupyter_console - - jupyterhub==5.1.0 + - jupyterhub==5.3.0 - nbconvert - nbval @@ -36,13 +36,13 @@ dependencies: - jupyterlab-pioneer - jupyter-ai - jupyterlab-favorites >=3.2.1 - - jupyter-scheduler >=2.5.2,<2.6.0 + - jupyter-scheduler >=2.8.0,<3.0.0 # >=2.8 due to https://github.com/conda-forge/jupyter_scheduler-feedstock/issues/46 # viz tools - param - python-graphviz - plotly >=5.0 - - ipympl + - ipympl >=0.9.6 - bokeh >=3.5.2 # testing, docs, linting @@ -59,9 +59,9 @@ dependencies: - pip: # vscode jupyterlab launcher - git+https://github.com/betatim/vscode-binder - - jupyterlab_nvdashboard==0.11.0 + - jupyterlab_nvdashboard==0.12.0 - argo-jupyter-scheduler==2024.6.1 - - jhub-apps==2024.12.1 + - jhub-apps==2025.2.1 - jupyterlab-nebari-mode==0.3.0 - jupyterlab-conda-store==2024.11.1 - jupyterlab-launchpad==1.0.3 From 51ed4c227ef2d492312f9b3165edd9b52a72d9b8 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 17:58:50 +0530 Subject: [PATCH 12/29] Delete docker/nebari-workflow-controller/apt.txt --- docker/nebari-workflow-controller/apt.txt | 26 ----------------------- 1 file changed, 26 deletions(-) delete mode 100644 docker/nebari-workflow-controller/apt.txt diff --git a/docker/nebari-workflow-controller/apt.txt b/docker/nebari-workflow-controller/apt.txt deleted file mode 100644 index 05d484cea..000000000 --- a/docker/nebari-workflow-controller/apt.txt +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright (c) Nebari Development Team. -# Distributed under the terms of the Modified BSD License. - -locales - -# assign uid/gid names -libnss-wrapper - -# utilities -wget -curl -htop -tree -zip -unzip - -# development utilities -git -openssh-client -tmux -xvfb - -# editors -nano -vim -emacs \ No newline at end of file From f5d32a87daaeefc7a930777f3cbf332c3872ebb3 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:03:19 +0530 Subject: [PATCH 13/29] Update install-code-server.sh from optimize dockerfile (#211) commit From 0c1bd93e73d22ef0dca3c7dd7324e95edc642750 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:04:08 +0530 Subject: [PATCH 14/29] Update install-code-server.sh from optimize dockerfile (#211) commit From 41e851a06d43902b5e7a2972cdad4c909af77c77 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:05:32 +0530 Subject: [PATCH 15/29] Update install-conda.sh from optimize dockerfile (#211) commit --- docker/scripts/install-conda.sh | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/docker/scripts/install-conda.sh b/docker/scripts/install-conda.sh index 43f018619..feda68b22 100644 --- a/docker/scripts/install-conda.sh +++ b/docker/scripts/install-conda.sh @@ -9,23 +9,28 @@ arch=$(uname -i) wget --quiet -O mambaforge.sh https://github.com/conda-forge/miniforge/releases/download/$MAMBAFORGE_VERSION/Mambaforge-Linux-$arch.sh if [[ $arch == "aarch64" ]]; then - echo "${MAMBAFORGE_AARCH64_SHA256} mambaforge.sh" >mambaforge.checksum + echo "${MAMBAFORGE_AARCH64_SHA256} mambaforge.sh" >mambaforge.checksum elif [[ $arch == "x86_64" ]]; then - echo "${MAMBAFORGE_X86_64_SHA256} mambaforge.sh" >mambaforge.checksum + echo "${MAMBAFORGE_X86_64_SHA256} mambaforge.sh" >mambaforge.checksum else - echo "Unsupported architecture: $arch" - exit 1 + echo "Unsupported architecture: $arch" + exit 1 fi echo $(sha256sum -c mambaforge.checksum) if [ $(sha256sum -c mambaforge.checksum | awk '{print $2}') != "OK" ]; then - echo Error when testing checksum - exit 1 + echo Error when testing checksum + exit 1 fi # Install Mamba and clean-up -sh ./mambaforge.sh -b -p /opt/conda +if [ -d "/opt/conda" ]; then + sh ./mambaforge.sh -b -u -p /opt/conda +else + sh ./mambaforge.sh -b -p /opt/conda +fi + rm mambaforge.sh mambaforge.checksum mamba --version From 82ec35ac42412f18c5f97c9242eca98ffd91130d Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:06:42 +0530 Subject: [PATCH 16/29] Update install-code-server.sh from optimize dockerfile (#211) commit From 1844d7d8f747c14aed6d5376dffc2e332a0e9651 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:07:19 +0530 Subject: [PATCH 17/29] Delete docker/scripts/install-apt-minimal.sh --- docker/scripts/install-apt-minimal.sh | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 docker/scripts/install-apt-minimal.sh diff --git a/docker/scripts/install-apt-minimal.sh b/docker/scripts/install-apt-minimal.sh deleted file mode 100644 index 297177dca..000000000 --- a/docker/scripts/install-apt-minimal.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) Nebari Development Team. -# Distributed under the terms of the Modified BSD License. - -apt-get update --fix-missing && - apt-get install -y wget bzip2 ca-certificates curl git && - apt-get clean && - rm -rf /var/lib/apt/lists/* /var/tmp/* /tmp/* From d6730bfa5ec32765eb8752381edba744044bcc00 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:07:59 +0530 Subject: [PATCH 18/29] Delete docker/scripts/install-apt.sh --- docker/scripts/install-apt.sh | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 docker/scripts/install-apt.sh diff --git a/docker/scripts/install-apt.sh b/docker/scripts/install-apt.sh deleted file mode 100644 index 6c142f46b..000000000 --- a/docker/scripts/install-apt.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash -# Copyright (c) Nebari Development Team. -# Distributed under the terms of the Modified BSD License. - -set -xe - -# Assumes apt packages installs packages in "$1" argument - -# ====== install apt packages ======== -apt-get update -apt-get install -y --no-install-recommends $(grep -vE "^\s*#" $1 | tr "\n" " ") - -# ========== cleanup apt ============= -apt-get clean -rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* From 7ca9de436d829a83bd31ec4c55b9d9915fe2d13f Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:10:09 +0530 Subject: [PATCH 19/29] Delete docker/Dockerfile.dask-worker --- docker/Dockerfile.dask-worker | 41 ----------------------------------- 1 file changed, 41 deletions(-) delete mode 100644 docker/Dockerfile.dask-worker diff --git a/docker/Dockerfile.dask-worker b/docker/Dockerfile.dask-worker deleted file mode 100644 index 693ecffa6..000000000 --- a/docker/Dockerfile.dask-worker +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Nebari Development Team. -# Distributed under the terms of the Modified BSD License. -# Usage: -# ------ -# -# To make a local build of the container, from the root directory: -# docker build -f Dockerfile.dask-worker -t nebari-dask-worker:latest . - -ARG BASE_IMAGE=ubuntu:20.04 -FROM $BASE_IMAGE -LABEL MAINTAINER="Nebari development team" - -COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh -RUN /opt/scripts/install-apt-minimal.sh - -COPY scripts/fix-permissions /opt/scripts/fix-permissions - -ENV MAMBAFORGE_VERSION 4.13.0-1 -ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 -ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c -SHELL ["/bin/bash", "-c"] - -ENV PATH=/opt/conda/bin:${PATH}:/opt/scripts - -# ============== base install =============== -COPY scripts/install-conda.sh /opt/scripts/install-conda.sh - -RUN /opt/scripts/install-conda.sh - -# ========== dask-worker install =========== -COPY dask-worker/environment.yaml /opt/dask-worker/environment.yaml -COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh -RUN /opt/scripts/install-conda-environment.sh /opt/dask-worker/environment.yaml 'false' - -# ========== Setup GPU Paths ============ -ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64 -ENV NVIDIA_PATH=/usr/local/nvidia/bin -ENV PATH="$NVIDIA_PATH:$PATH" - -COPY dask-worker /opt/dask-worker -RUN /opt/dask-worker/postBuild From d9e352fd851fe4ea6d6a3c1d5bc5f472c41afa2b Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:10:19 +0530 Subject: [PATCH 20/29] Delete docker/Dockerfile.jupyterhub --- docker/Dockerfile.jupyterhub | 41 ------------------------------------ 1 file changed, 41 deletions(-) delete mode 100644 docker/Dockerfile.jupyterhub diff --git a/docker/Dockerfile.jupyterhub b/docker/Dockerfile.jupyterhub deleted file mode 100644 index d1c696f94..000000000 --- a/docker/Dockerfile.jupyterhub +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) Nebari Development Team. -# Distributed under the terms of the Modified BSD License. -# Usage: -# ------ -# -# To make a local build of the container, from the root directory: -# docker build -f Dockerfile.jupyterhub -t nebari-jupyterhub:latest . - -FROM ubuntu:20.04 -LABEL MAINTAINER="Nebari development team" - -COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh -RUN /opt/scripts/install-apt-minimal.sh - -COPY scripts/fix-permissions /opt/scripts/fix-permissions - -ENV MAMBAFORGE_VERSION 4.13.0-1 -ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 -ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c -SHELL ["/bin/bash", "-c"] - -ENV PATH="/opt/conda/bin:$PATH:/opt/scripts" - -# ============== base install =============== -COPY scripts/install-conda.sh /opt/scripts/install-conda.sh -RUN /opt/scripts/install-conda.sh - -# ========== jupyterhub install =========== -COPY jupyterhub/environment.yaml /opt/jupyterhub/environment.yaml -COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh -RUN /opt/scripts/install-conda-environment.sh /opt/jupyterhub/environment.yaml 'false' - -COPY jupyterhub /opt/jupyterhub -RUN /opt/jupyterhub/postBuild - -WORKDIR /srv/jupyterhub - -# So we can actually write a db file here -RUN fix-permissions /srv/jupyterhub - -CMD ["jupyterhub", "--config", "/usr/local/etc/jupyterhub/jupyterhub_config.py"] From 686d04af0c42966d1f13aa4395d89c218d168994 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:10:30 +0530 Subject: [PATCH 21/29] Delete docker/Dockerfile.jupyterlab --- docker/Dockerfile.jupyterlab | 70 ------------------------------------ 1 file changed, 70 deletions(-) delete mode 100644 docker/Dockerfile.jupyterlab diff --git a/docker/Dockerfile.jupyterlab b/docker/Dockerfile.jupyterlab deleted file mode 100644 index 06a2d9fcf..000000000 --- a/docker/Dockerfile.jupyterlab +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright (c) Nebari Development Team. -# Distributed under the terms of the Modified BSD License. -# Usage: -# ------ -# -# To make a local build of the container, from the root directory: -# docker build -f Dockerfile.jupyterlab -t nebari-jupyterlab:latest . - -ARG BASE_IMAGE=ubuntu:20.04 -FROM $BASE_IMAGE -LABEL MAINTAINER="Nebari development team" - -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -RUN chmod -R a-w ~ -COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh -RUN /opt/scripts/install-apt-minimal.sh - -COPY scripts/fix-permissions /opt/scripts/fix-permissions - -ENV MAMBAFORGE_VERSION 4.13.0-1 -ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 -ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c -SHELL ["/bin/bash", "-c"] -ENV CONDA_DIR=/opt/conda \ - DEFAULT_ENV=default -# Set timezone -ENV TZ=America/Chicago -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -# Set PATH for Dockerfile so that conda works and some useful scripts are -# available. Any changes intended to propagate to runtime containers should be -# set in /etc/profile.d (see setup_shell_behavior.sh) -ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/bin:/opt/conda/bin:${PATH}:/opt/scripts - -# ============= base install =============== -# install conda -COPY scripts/install-conda.sh /opt/scripts/install-conda.sh -RUN echo "${SHELL}"; env; cat ~/.bashrc; cat ~/.profile ; /opt/scripts/install-conda.sh - -# ========== jupyterlab install ============ -COPY jupyterlab/apt.txt /opt/jupyterlab/apt.txt -COPY scripts/install-apt.sh /opt/scripts/install-apt.sh -RUN /opt/scripts/install-apt.sh /opt/jupyterlab/apt.txt - -# Install extra packages (require custom package repository) -COPY scripts/install-gitlfs.sh /opt/scripts/install-gitlfs.sh -RUN /opt/scripts/install-gitlfs.sh - -ARG SKIP_CONDA_SOLVE=no -COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh -COPY jupyterlab/environment.yaml /opt/jupyterlab/environment.yaml -RUN \ - if [ "${SKIP_CONDA_SOLVE}" != "no" ];then \ - ENV_FILE=/opt/jupyterlab/conda-linux-64.lock ; \ - else \ - ENV_FILE=/opt/jupyterlab/environment.yaml ; \ - fi ; \ - /opt/scripts/install-conda-environment.sh "${ENV_FILE}" 'true' - -# ========== code-server install ============ -ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/share/code-server/bin:${PATH} -COPY scripts/install-code-server.sh /opt/scripts/install-code-server.sh - -COPY jupyterlab /opt/jupyterlab -RUN /opt/jupyterlab/postBuild - -# ========== Setup GPU Paths ============ -ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64 -ENV NVIDIA_PATH=/usr/local/nvidia/bin -ENV PATH="$NVIDIA_PATH:$PATH" From d2ec3ecc0c612f1331d1f5e193b31903cfd4f759 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:10:38 +0530 Subject: [PATCH 22/29] Delete docker/Dockerfile.workflow-controller --- docker/Dockerfile.workflow-controller | 61 --------------------------- 1 file changed, 61 deletions(-) delete mode 100644 docker/Dockerfile.workflow-controller diff --git a/docker/Dockerfile.workflow-controller b/docker/Dockerfile.workflow-controller deleted file mode 100644 index 968523936..000000000 --- a/docker/Dockerfile.workflow-controller +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) Nebari Development Team. -# Distributed under the terms of the Modified BSD License. -# Usage: -# ------ -# -# To make a local build of the container, from the root directory: -# docker build -f Dockerfile.workflow-controller -t nebari-workflow-controller:latest . - -ARG BASE_IMAGE=ubuntu:20.04 -FROM $BASE_IMAGE -LABEL MAINTAINER="Nebari development team" - -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -RUN chmod -R a-w ~ -COPY scripts/install-apt-minimal.sh /opt/scripts/install-apt-minimal.sh -RUN /opt/scripts/install-apt-minimal.sh - -COPY scripts/fix-permissions /opt/scripts/fix-permissions - -ENV MAMBAFORGE_VERSION 4.13.0-1 -ENV MAMBAFORGE_AARCH64_SHA256 69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 -ENV MAMBAFORGE_X86_64_SHA256 412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c -SHELL ["/bin/bash", "-c"] -ENV CONDA_DIR=/opt/conda \ - DEFAULT_ENV=default -# Set timezone -ENV TZ=America/Chicago -RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone - -# Set PATH for Dockerfile so that conda works and some useful scripts are -# available. Any changes intended to propagate to runtime containers should be -# set in /etc/profile.d (see setup_shell_behavior.sh) -ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/bin:/opt/conda/bin:${PATH}:/opt/scripts - -# ============= base install =============== -# install conda -COPY scripts/install-conda.sh /opt/scripts/install-conda.sh -RUN echo "${SHELL}"; env; cat ~/.bashrc; cat ~/.profile ; /opt/scripts/install-conda.sh - -# ========== nebari-workflow-controller install ============ -COPY scripts/install-apt.sh /opt/scripts/install-apt.sh -COPY nebari-workflow-controller/apt.txt /opt/nebari-workflow-controller/apt.txt -RUN /opt/scripts/install-apt.sh - -# uncomment to install dev dependencies -# RUN /opt/scripts/install-apt.sh /opt/nebari-workflow-controller/apt.txt - -ARG SKIP_CONDA_SOLVE=no -COPY scripts/install-conda-environment.sh /opt/scripts/install-conda-environment.sh -COPY nebari-workflow-controller/environment.yaml /opt/nebari-workflow-controller/environment.yaml -RUN \ - if [ "${SKIP_CONDA_SOLVE}" != "no" ];then \ - ENV_FILE=/opt/nebari-workflow-controller/conda-linux-64.lock ; \ - else \ - ENV_FILE=/opt/nebari-workflow-controller/environment.yaml ; \ - fi ; \ - /opt/scripts/install-conda-environment.sh "${ENV_FILE}" 'true' - -COPY nebari-workflow-controller /opt/nebari-workflow-controller - -CMD ["python", "-m", "nebari_workflow_controller"] \ No newline at end of file From 02f9a793dcf29bffd129e6362c802206b7719bca Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:11:31 +0530 Subject: [PATCH 23/29] Created Dockerfile from Update base images to use Ubuntu 24.04 (#213) --- docker/Dockerfile | 158 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 000000000..b634b69fc --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,158 @@ +FROM ubuntu:24.04 AS builder +LABEL MAINTAINER="Nebari development team" + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ + wget \ + bzip2 \ + ca-certificates \ + curl \ + git + +COPY scripts /opt/scripts + +ENV MAMBAFORGE_VERSION=4.13.0-1 \ + MAMBAFORGE_AARCH64_SHA256=69e3c90092f61916da7add745474e15317ed0dc6d48bfe4e4c90f359ba141d23 \ + MAMBAFORGE_X86_64_SHA256=412b79330e90e49cf7e39a7b6f4752970fcdb8eb54b1a45cc91afe6777e8518c \ + PATH=/opt/conda/bin:${PATH}:/opt/scripts + + +RUN /opt/scripts/install-conda.sh + + + +# ========== dask-worker install =========== +FROM builder AS dask-worker +COPY dask-worker/environment.yaml /opt/dask-worker/environment.yaml +RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \ + --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + /opt/scripts/install-conda-environment.sh /opt/dask-worker/environment.yaml 'false' + +ENV LD_LIBRARY_PATH=/usr/local/nvidia/lib64 +ENV NVIDIA_PATH=/usr/local/nvidia/bin +ENV PATH="$NVIDIA_PATH:$PATH" + +COPY dask-worker /opt/dask-worker +RUN /opt/dask-worker/postBuild + + + + + +# ========== jupyterhub install =========== +FROM builder AS jupyterhub +COPY jupyterhub/environment.yaml /opt/jupyterhub/environment.yaml +RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \ + --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + /opt/scripts/install-conda-environment.sh /opt/jupyterhub/environment.yaml 'false' + +COPY jupyterhub /opt/jupyterhub +RUN /opt/jupyterhub/postBuild + +WORKDIR /srv/jupyterhub + +# So we can actually write a db file here +RUN fix-permissions /srv/jupyterhub + +CMD ["jupyterhub", "--config", "/usr/local/etc/jupyterhub/jupyterhub_config.py"] + + + + +# ========== jupyterlab base =========== +FROM builder AS intermediate +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 \ + CONDA_DIR=/opt/conda \ + DEFAULT_ENV=default +RUN chmod -R a-w ~ +ENV TZ=UTC \ + PATH=/opt/conda/envs/${DEFAULT_ENV}/bin:/opt/conda/bin:${PATH}:/opt/scripts +# Set timezone +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ + locales \ + libnss-wrapper \ + htop \ + tree \ + zip \ + unzip \ + openssh-client \ + tmux \ + xvfb \ + nano \ + vim \ + emacs + + +# ========== jupyterlab install =========== +FROM intermediate AS jupyterlab +ENV CONDA_DIR=/opt/conda \ + DEFAULT_ENV=default \ + LD_LIBRARY_PATH=/usr/local/nvidia/lib64 \ + NVIDIA_PATH=/usr/local/nvidia/bin + +ENV PATH="$NVIDIA_PATH:$PATH" + +RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ + --mount=type=cache,target=/var/lib/apt,sharing=locked \ + apt-get update && apt-get install -y --no-install-recommends \ + zsh \ + neovim \ + libgl1 \ + libglx-mesa0 \ + libxrandr2 \ + libxss1 \ + libxcursor1 \ + libxcomposite1 \ + libasound2t64 \ + libxi6 \ + libxtst6 \ + libfontconfig1 \ + libxrender1 \ + libosmesa6 \ + gnupg \ + pinentry-curses \ + git-lfs + +ARG SKIP_CONDA_SOLVE=no +COPY jupyterlab/environment.yaml /opt/jupyterlab/environment.yaml +RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \ + --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + if [ "${SKIP_CONDA_SOLVE}" != "no" ];then \ + ENV_FILE=/opt/jupyterlab/conda-linux-64.lock ; \ + else \ + ENV_FILE=/opt/jupyterlab/environment.yaml ; \ + fi ; \ + /opt/scripts/install-conda-environment.sh "${ENV_FILE}" 'true' + +# ========== code-server install ============ +ENV PATH=/opt/conda/envs/${DEFAULT_ENV}/share/code-server/bin:${PATH} + +COPY jupyterlab /opt/jupyterlab +RUN /opt/jupyterlab/postBuild + + + + + +# ========== nebari-workflow-controller install ============ +FROM intermediate AS workflow-controller + +ARG SKIP_CONDA_SOLVE=no +COPY nebari-workflow-controller/environment.yaml /opt/nebari-workflow-controller/environment.yaml +RUN --mount=type=cache,target=/opt/conda/pkgs,sharing=locked \ + --mount=type=cache,target=/root/.cache/pip,sharing=locked \ + if [ "${SKIP_CONDA_SOLVE}" != "no" ];then \ + ENV_FILE=/opt/nebari-workflow-controller/conda-linux-64.lock ; \ + else \ + ENV_FILE=/opt/nebari-workflow-controller/environment.yaml ; \ + fi ; \ + /opt/scripts/install-conda-environment.sh "${ENV_FILE}" 'true' + +COPY nebari-workflow-controller /opt/nebari-workflow-controller + +CMD ["python", "-m", "nebari_workflow_controller"] From 4a91c3eb09e74f081363875a1738fdd218771bdf Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:12:33 +0530 Subject: [PATCH 24/29] Created makefile from optimize dockerfile (#211) commit --- docker/makefile | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 docker/makefile diff --git a/docker/makefile b/docker/makefile new file mode 100644 index 000000000..c22d0558d --- /dev/null +++ b/docker/makefile @@ -0,0 +1,27 @@ +IMAGES := jupyterhub jupyterlab dask-worker workflow-controller +DOCKERFILE := Dockerfile +CONTEXT := . + +.PHONY: all $(IMAGES) clean + +# Build all images +all: $(IMAGES) + +# Build individual images +jupyterhub: + docker build -t nebari-dev/nebari-docker-images:nebari-jupyterhub -f $(DOCKERFILE) $(CONTEXT) --target jupyterhub + +jupyterlab: + docker build -t nebari-dev/nebari-docker-images:nebari-jupyterlab -f $(DOCKERFILE) $(CONTEXT) --target jupyterlab + +dask-worker: + docker build -t nebari-dev/nebari-docker-images:nebari-dask-worker -f $(DOCKERFILE) $(CONTEXT) --target dask-worker + +workflow-controller: + docker build -t nebari-dev/nebari-docker-images:nebari-workflow-controller -f $(DOCKERFILE) $(CONTEXT) --target workflow-controller + +# Clean up images +clean: + @for image in $(IMAGES); do \ + docker rmi nebari-dev/nebari-docker-images:nebari-$$image; \ + done From 5b87da4c6b558bc3dc4172ada6fe61259a06056c Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:13:20 +0530 Subject: [PATCH 25/29] Updated README.md from optimize dockerfile (#211) commit --- docker/README.md | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/docker/README.md b/docker/README.md index cee5c7f28..5a0553c88 100644 --- a/docker/README.md +++ b/docker/README.md @@ -62,25 +62,39 @@ Whether you want to contribute to this project or whether you wish use these ima From the repository's root folder, you can build these images locally by running the listed commands on your terminal. -- For [JupyterLab](Dockerfile.jupyterlab): +- To build nebari-jupyterlab ```shell - docker build -f Dockerfile.jupyterlab \ - -t nebari-jupyterlab:latest . + make jupyterlab ``` -- For [JupyterHub](Dockerfile.jupyterhub): +- To build nebari-jupyterhub ```shell - docker build -f Dockerfile.jupyterhub \ - -t nebari-jupyterhub:latest . + make jupyterhub ``` -- For [Dask-Worker](Dockerfile.dask-worker): +- To build nebari-dask-worker ```shell - docker build -f Dockerfile.dask-gateway \ - -t nebari-dask-gateway:latest . + make dask-worker + ``` + +- To build nebari-workflow-controller + + ```shell + make workflow-controller + ``` + +- To build all of the images + + ```shell + make all + ``` +- To delete built images + + ```shell + make clean ``` > **NOTE** From 2457d6a92b3bdfd11bb6ebb38953f826604e625c Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:16:29 +0530 Subject: [PATCH 26/29] Created build_push_docker.yaml from Update base images to use Ubuntu 24.04 (#213) commit --- .github/workflows/build_push_docker.yaml | 123 +++++++++++++++++++++++ 1 file changed, 123 insertions(+) create mode 100644 .github/workflows/build_push_docker.yaml diff --git a/.github/workflows/build_push_docker.yaml b/.github/workflows/build_push_docker.yaml new file mode 100644 index 000000000..af4ff271e --- /dev/null +++ b/.github/workflows/build_push_docker.yaml @@ -0,0 +1,123 @@ +# Build and push images to: +# GitHub Container Registry (ghcr.io) +# Red Hat Container Registry (quay.io) +name: "Build Docker Images" + +on: + workflow_dispatch: null + push: + branches: + - "*" + paths: + - "Dockerfile" + - "dask-worker/*" + - "jupyterhub/*" + - "jupyterlab/*" + - "nebari-workflow-controller/*" + + - "scripts/*" + + - ".github/workflows/build-push-docker.yaml" + tags: + - "*" + +env: + DOCKER_ORG: nebari + GPU_BASE_IMAGE: nvidia/cuda:12.8.1-base-ubuntu24.04 + GPU_IMAGE_SUFFIX: gpu + BASE_IMAGE: ubuntu:24.04 + +permissions: + contents: read + packages: write + id-token: write + security-events: write + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-images: + name: "Build Docker Images" + runs-on: ubuntu-latest + strategy: + matrix: + dockerfile: + - jupyterlab + - jupyterhub + - dask-worker + - workflow-controller + platform: + - gpu + - cpu + exclude: + # excludes JupyterHub/GPU, Workflow Controller/GPU + - dockerfile: jupyterhub + platform: gpu + - dockerfile: workflow-controller + platform: gpu + + steps: + - name: "Checkout Repository ๐Ÿ›Ž๏ธ" + uses: actions/checkout@v3 + + - name: "Set up Docker Buildx ๐Ÿ› ๏ธ" + uses: docker/setup-buildx-action@v2 + + - name: "Login to GitHub Container Registry ๐Ÿ”" + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.repository_owner }} + password: ${{ secrets.BOT_GHCR_TOKEN }} + + - name: "Login to Quay Container Registry ๐Ÿ”" + uses: docker/login-action@v2 + with: + registry: quay.io + username: ${{ secrets.QUAY_USERNAME }} + password: ${{ secrets.QUAY_TOKEN }} + + - name: "Set BASE_IMAGE and Image Suffix ๐Ÿ“ท" + if: ${{ matrix.platform == 'gpu' }} + run: | + echo "GPU Platform Matrix" + echo "BASE_IMAGE=$GPU_BASE_IMAGE" >> $GITHUB_ENV + echo "IMAGE_SUFFIX=-$GPU_IMAGE_SUFFIX" >> $GITHUB_ENV + + - name: "Generate Docker images tags ๐Ÿท๏ธ" + id: meta + uses: docker/metadata-action@v4 + with: + images: | + "quay.io/${{ env.DOCKER_ORG }}/nebari-${{ matrix.dockerfile }}${{ env.IMAGE_SUFFIX }}" + "ghcr.io/${{ github.repository_owner }}/nebari-${{ matrix.dockerfile }}${{ env.IMAGE_SUFFIX }}" + tags: | + # branch event -> e.g. `main-f0f6994-20221001` + type=ref, event=branch, suffix=-{{sha}}-{{date 'YYYYMMDD'}} + # needed for integration tests + type=ref, event=branch + # on tag push -> e.g. `2022.10.1` + type=ref, event=tag + + - name: "Inspect image dir tree ๐Ÿ”" + run: | + sudo apt-get install tree + tree . + + - name: "Build docker images ๐Ÿณ" + uses: docker/build-push-action@v3 + with: + context: . + file: "Dockerfile" + target: ${{ matrix.dockerfile }} + tags: ${{ steps.meta.outputs.tags }} + push: ${{ github.event_name != 'pull_request' }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BASE_IMAGE=${{ env.BASE_IMAGE }} + platforms: linux/amd64,linux/arm64 From 358c8d014500159193206c1e1d4a87e5a3ef13cd Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:17:10 +0530 Subject: [PATCH 27/29] Create test_images.yaml from Update base images to use Ubuntu 24.04 (#213) commit --- .github/workflows/test_images.yaml | 77 ++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) create mode 100644 .github/workflows/test_images.yaml diff --git a/.github/workflows/test_images.yaml b/.github/workflows/test_images.yaml new file mode 100644 index 000000000..d3a6a1962 --- /dev/null +++ b/.github/workflows/test_images.yaml @@ -0,0 +1,77 @@ +name: Test Docker images + +on: + pull_request: + paths: + - "Dockerfile.*" + + - "dask-worker/*" + - "jupyterhub/*" + - "jupyterlab/*" + + - "scripts/*" + + - ".github/workflows/build-push-docker.yaml" + - ".github/workflows/test-images.yaml" + +env: + DOCKER_ORG: nebari + GITHUB_SHA: ${{ github.sha }} + GPU_BASE_IMAGE: nvidia/cuda:12.8.1-base-ubuntu24.04 + GPU_IMAGE_SUFFIX: gpu + BASE_IMAGE: ubuntu:24.04 + +# https://docs.github.com/en/actions/using-jobs/using-concurrency +concurrency: + # only cancel in-progress jobs or runs for the current workflow - matches against branch & tags + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + build-test-images: + runs-on: ubuntu-latest + strategy: + matrix: + dockerfile: + - jupyterlab + - jupyterhub + - dask-worker + platform: + - gpu + - cpu + exclude: + # excludes JupyterHub/GPU + - dockerfile: jupyterhub + platform: gpu + steps: + - name: Checkout Repository ๐Ÿ›Ž + uses: actions/checkout@v3 + + - name: Lint Dockerfiles ๐Ÿ” + uses: jbergstroem/hadolint-gh-action@v1 + with: + dockerfile: Dockerfile + output_format: tty + error_level: 0 + + - name: "Set BASE_IMAGE and Image Suffix ๐Ÿ“ท" + if: ${{ matrix.platform == 'gpu' }} + run: | + echo "GPU Platform Matrix" + echo "BASE_IMAGE=$GPU_BASE_IMAGE" >> $GITHUB_ENV + echo "IMAGE_SUFFIX=-$GPU_IMAGE_SUFFIX" >> $GITHUB_ENV + + - name: "Set up Docker Buildx ๐Ÿ› ๏ธ" + uses: docker/setup-buildx-action@v2 + + - name: Build Image ๐Ÿ›  + uses: docker/build-push-action@v3 + with: + context: . + file: "Dockerfile" + target: ${{ matrix.dockerfile }} + push: false + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: BASE_IMAGE=${{ env.BASE_IMAGE }} + platforms: linux/amd64,linux/arm64 From f7a5650ff96460bbd6cd21002462dc678d2fec50 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Fri, 20 Jun 2025 18:18:31 +0530 Subject: [PATCH 28/29] Created docker_trivy.yaml from Set up Trivy config scanning for Dockerfile misconfigurations (#214) commit --- .github/workflows/docker_trivy.yaml | 40 +++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/docker_trivy.yaml diff --git a/.github/workflows/docker_trivy.yaml b/.github/workflows/docker_trivy.yaml new file mode 100644 index 000000000..5644a3d5e --- /dev/null +++ b/.github/workflows/docker_trivy.yaml @@ -0,0 +1,40 @@ +name: Code Scanning + +on: + push: + branches: [ "main"] + pull_request: + # The branches below must be a subset of the branches above + branches: [ "main" ] + +permissions: + contents: read + +jobs: + SAST: + permissions: + contents: read # for actions/checkout to fetch code + security-events: write # for github/codeql-action/upload-sarif to upload SARIF results + actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status + name: Trivy config Scan + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Run Trivy vulnerability scanner in config mode + uses: aquasecurity/trivy-action@master + with: + scan-type: 'config' + hide-progress: true + format: 'sarif' + output: 'trivy-results.sarif' + ignore-unfixed: true + severity: 'CRITICAL,HIGH' + limit-severities-for-sarif: true + + - name: Upload Trivy scan results to GitHub Security tab + uses: github/codeql-action/upload-sarif@v3 + if: always() + with: + sarif_file: 'trivy-results.sarif' From b71ceab2ec66009f32b30942ef63f65f80475188 Mon Sep 17 00:00:00 2001 From: Rakesh Bajpayee <153479979+Helion55@users.noreply.github.com> Date: Tue, 1 Jul 2025 19:09:00 +0530 Subject: [PATCH 29/29] Delete .github/ISSUE_TEMPLATE/RFD.md --- .github/ISSUE_TEMPLATE/RFD.md | 52 ----------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/RFD.md diff --git a/.github/ISSUE_TEMPLATE/RFD.md b/.github/ISSUE_TEMPLATE/RFD.md deleted file mode 100644 index ab29a17d4..000000000 --- a/.github/ISSUE_TEMPLATE/RFD.md +++ /dev/null @@ -1,52 +0,0 @@ ---- -name: "Request for Discussion (RFD) ๐Ÿ’ฌ" -about: Open discussion about a feature or design project in Nebari. -labels: - - "type: RFD" -title: "RFD - Title" ---- - -<!-- Example of when and how to create a RFD or RFC (request for comments) --> -<!-- https://gitpod.notion.site/Decision-Making-RFCs-eb4a57f3a34f40f1afbd95e05322af70 --> - -<!-- Use this guide to set the status: Draft ๐Ÿšง / Open for comments ๐Ÿ’ฌ/ Accepted โœ… /Implemented ๐Ÿš€/ Obsolete ๐Ÿ—ƒ --> - -| Status | Draft ๐Ÿšง / Open for comments ๐Ÿ’ฌ/ Accepted โœ… /Implemented ๐Ÿš€/ Obsolete ๐Ÿ—ƒ | -| ----------------- | ------------------------------------------------------------------------ | -| Author(s) | GitHub handle | -| Date Created | dd-MM-YYY | -| Date Last updated | dd-MM-YYY | -| Decision deadline | dd-MM-YYY | - -# Title - -## Summary - -<!-- What are we trying to solve here? Try and make this concise--> - -## User benefit - -<!-- How will users (or other contributors) benefit from this work? What would be the headline in the release notes or blog post? --> - -## Design Proposal - -<!--This is the meat of the document, where you explain your proposal. - -Explain the design in enough detail for somebody familiar with the project to understand. Include examples of how the feature/implementation will work. Feel free to add schematics, drawings or other supporting visual material. --> - -### Alternatives or approaches considered (if any) - -<!-- Make sure to discuss the relative merits of alternatives to your proposal. --> - -### Best practices - -<!-- Does this proposal change best practices for some aspect of using/developing JupyterLab or other project? How will these changes be communicated/enforced? - --> - -### User impact - -<!-- What are the user-facing changes? How will this feature be rolled out? --> - -## Unresolved questions - -<!-- Seed this with open questions you require feedback on from the RFD process. -->