Skip to content

Commit f86dbe6

Browse files
jkulhanekginazhouhuiwubrentyi
authored
Update dockerfile and add docker build action (#3283)
* Use github docker registry * Fix DDP train for GPU in exclusive mode * Improve docker image - compile gsplat, decrease image size * Drop unrelated change * Add build docker image action * Rename build docker image action * nit * Remove commented line from Dockerfile * Fix dockerfile when explicit source is specified * Fix failing dynamo build for torch.compile * Lock dockerfile and tcnn versions * Add `torch.cuda.is_available()` condition * Drop set_cuda_device * Fix build docker image github action * Docker build save disk space * Set MAX_JOBS to limit resource usage for docker build * Try bumping `MAX_JOBS` 2 => 4 * Install fixed gsplat version from nerfstudio's pyproject.toml * Update docs * Finish docker build action * Fix ignores push when PR --------- Co-authored-by: Gina Wu <ginawu98@gmail.com> Co-authored-by: Brent Yi <yibrenth@gmail.com>
1 parent 96b7fe2 commit f86dbe6

File tree

3 files changed

+188
-189
lines changed

3 files changed

+188
-189
lines changed

‎.github/workflows/build_docker_image.yml

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
name: Build Docker Image
2+
on:
3+
workflow_dispatch:
4+
workflow_call:
5+
pull_request:
6+
push:
7+
branches:
8+
- main
9+
- master
10+
tags:
11+
- 'v*'
12+
env:
13+
REGISTRY: ghcr.io
14+
IMAGE_NAME: ${{ github.repository }}
15+
jobs:
16+
build-and-publish-docker-image:
17+
runs-on: ubuntu-latest
18+
name: build-and-publish-docker-image
19+
permissions:
20+
packages: write
21+
contents: read
22+
attestations: write
23+
id-token: write
24+
steps:
25+
- uses: actions/checkout@v4
26+
- name: Login to GitHub Container Registry
27+
uses: docker/login-action@v3
28+
with:
29+
registry: ghcr.io
30+
username: ${{ github.repository_owner }}
31+
password: ${{ secrets.GITHUB_TOKEN }}
32+
- name: Extract metadata (tags, labels) for Docker
33+
id: meta
34+
uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
35+
with:
36+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
37+
tags: |
38+
type=ref,event=branch
39+
type=ref,event=pr
40+
type=semver,pattern={{version}}
41+
type=semver,pattern={{major}}.{{minor}}
42+
- name: Free root space
43+
uses: almahmoud/free-root-space@main
44+
with:
45+
remove-gcc: false
46+
remove-cplusplus: false
47+
- name: Build and push Docker image
48+
id: push
49+
uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
50+
with:
51+
context: .
52+
file: ./Dockerfile
53+
push: ${{ github.event_name != 'pull_request' }}
54+
tags: ${{ steps.meta.outputs.tags }}
55+
labels: ${{ steps.meta.outputs.labels }}
56+
- name: Generate artifact attestation
57+
uses: actions/attest-build-provenance@v1
58+
with:
59+
subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME}}
60+
subject-digest: ${{ steps.push.outputs.digest }}
61+
push-to-registry: ${{ github.event_name != 'pull_request' }}
62+

‎Dockerfile

Lines changed: 122 additions & 174 deletions
Original file line numberDiff line numberDiff line change
@@ -1,185 +1,133 @@
1-
ARG CUDA_VERSION=11.8.0
2-
ARG OS_VERSION=22.04
3-
# Define base image.
4-
FROM nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${OS_VERSION}
5-
ARG CUDA_VERSION
6-
ARG OS_VERSION
1+
# syntax=docker/dockerfile:1
2+
ARG UBUNTU_VERSION=22.04
3+
ARG NVIDIA_CUDA_VERSION=11.8.0
4+
# CUDA architectures, required by Colmap and tiny-cuda-nn. Use >= 8.0 for faster TCNN.
5+
ARG CUDA_ARCHITECTURES="90;89;86;80;75;70;61"
6+
ARG NERFSTUDIO_VERSION=""
7+
8+
# Pull source either provided or from git.
9+
FROM scratch as source_copy
10+
ONBUILD COPY . /tmp/nerfstudio
11+
FROM alpine/git as source_no_copy
12+
ARG NERFSTUDIO_VERSION
13+
ONBUILD RUN git clone --branch ${NERFSTUDIO_VERSION} --recursive https://github.com/nerfstudio-project/nerfstudio.git /tmp/nerfstudio
14+
ARG NERFSTUDIO_VERSION
15+
FROM source_${NERFSTUDIO_VERSION:+no_}copy as source
16+
17+
FROM nvidia/cuda:${NVIDIA_CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION} as builder
18+
ARG CUDA_ARCHITECTURES
19+
ARG NVIDIA_CUDA_VERSION
20+
ARG UBUNTU_VERSION
721

8-
# Define username, user uid and gid
9-
ARG USERNAME=user
10-
ARG USER_UID=1000
11-
ARG USER_GID=$USER_UID
12-
13-
# metainformation
14-
LABEL org.opencontainers.image.version = "0.1.18"
15-
LABEL org.opencontainers.image.source = "https://github.com/nerfstudio-project/nerfstudio"
16-
LABEL org.opencontainers.image.licenses = "Apache License 2.0"
17-
LABEL org.opencontainers.image.base.name="docker.io/library/nvidia/cuda:${CUDA_VERSION}-devel-ubuntu${OS_VERSION}"
18-
19-
# Variables used at build time.
20-
## CUDA architectures, required by Colmap and tiny-cuda-nn.
21-
## NOTE: All commonly used GPU architectures are included and supported here. To speedup the image build process remove all architectures but the one of your explicit GPU. Find details here: https://developer.nvidia.com/cuda-gpus (8.6 translates to 86 in the line below) or in the docs.
22-
ARG CUDA_ARCHITECTURES=90;89;86;80;75;70;61;52;37
23-
24-
# Set environment variables.
25-
## Set non-interactive to prevent asking for user inputs blocking image creation.
2622
ENV DEBIAN_FRONTEND=noninteractive
27-
## Set timezone as it is required by some packages.
28-
ENV TZ=Europe/Berlin
29-
## CUDA Home, required to find CUDA in some packages.
30-
ENV CUDA_HOME="/usr/local/cuda"
31-
32-
# Install required apt packages and clear cache afterwards.
23+
ENV QT_XCB_GL_INTEGRATION=xcb_egl
3324
RUN apt-get update && \
34-
apt-get install -y --no-install-recommends \
35-
build-essential \
36-
cmake \
37-
curl \
38-
ffmpeg \
39-
git \
40-
libatlas-base-dev \
41-
libboost-filesystem-dev \
42-
libboost-graph-dev \
43-
libboost-program-options-dev \
44-
libboost-system-dev \
45-
libboost-test-dev \
46-
libhdf5-dev \
47-
libcgal-dev \
48-
libeigen3-dev \
49-
libflann-dev \
50-
libfreeimage-dev \
51-
libgflags-dev \
52-
libglew-dev \
53-
libgoogle-glog-dev \
54-
libmetis-dev \
55-
libprotobuf-dev \
56-
libqt5opengl5-dev \
57-
libsqlite3-dev \
58-
libsuitesparse-dev \
59-
nano \
60-
protobuf-compiler \
61-
python-is-python3 \
62-
python3.10-dev \
63-
python3-pip \
64-
qtbase5-dev \
65-
sudo \
66-
vim-tiny \
67-
wget && \
68-
rm -rf /var/lib/apt/lists/*
69-
70-
71-
# Install GLOG (required by ceres).
72-
RUN git clone --branch v0.6.0 https://github.com/google/glog.git --single-branch && \
73-
cd glog && \
74-
mkdir build && \
75-
cd build && \
76-
cmake .. && \
77-
make -j `nproc` && \
78-
make install && \
79-
cd ../.. && \
80-
rm -rf glog
81-
# Add glog path to LD_LIBRARY_PATH.
82-
ENV LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/usr/local/lib"
83-
84-
# Install Ceres-solver (required by colmap).
85-
RUN git clone --branch 2.1.0 https://ceres-solver.googlesource.com/ceres-solver.git --single-branch && \
86-
cd ceres-solver && \
87-
git checkout $(git describe --tags) && \
88-
mkdir build && \
89-
cd build && \
90-
cmake .. -DBUILD_TESTING=OFF -DBUILD_EXAMPLES=OFF && \
91-
make -j `nproc` && \
92-
make install && \
93-
cd ../.. && \
94-
rm -rf ceres-solver
95-
96-
# Install colmap.
97-
RUN git clone --branch 3.8 https://github.com/colmap/colmap.git --single-branch && \
25+
apt-get install -y --no-install-recommends --no-install-suggests \
26+
git \
27+
cmake \
28+
ninja-build \
29+
build-essential \
30+
libboost-program-options-dev \
31+
libboost-filesystem-dev \
32+
libboost-graph-dev \
33+
libboost-system-dev \
34+
libeigen3-dev \
35+
libflann-dev \
36+
libfreeimage-dev \
37+
libmetis-dev \
38+
libgoogle-glog-dev \
39+
libgtest-dev \
40+
libsqlite3-dev \
41+
libglew-dev \
42+
qtbase5-dev \
43+
libqt5opengl5-dev \
44+
libcgal-dev \
45+
libceres-dev \
46+
python3.10-dev \
47+
python3-pip
48+
49+
# Build and install COLMAP.
50+
RUN git clone https://github.com/colmap/colmap.git && \
9851
cd colmap && \
52+
git checkout "3.9.1" && \
9953
mkdir build && \
10054
cd build && \
101-
cmake .. -DCUDA_ENABLED=ON \
102-
-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES} && \
103-
make -j `nproc` && \
104-
make install && \
105-
cd ../.. && \
106-
rm -rf colmap
107-
108-
# Create non root user, add it to custom group and setup environment.
109-
RUN groupadd --gid $USER_GID $USERNAME \
110-
&& useradd --uid $USER_UID --gid $USER_GID -m $USERNAME -d /home/${USERNAME} --shell /usr/bin/bash
111-
# OPTIONAL
112-
# If sudo privilages are not required comment below line
113-
# Create simple password for user and add it to sudo group
114-
# Update group so that it is not required to type password for commands: apt update/upgrade/install/remove
115-
RUN echo "${USERNAME}:password" | chpasswd \
116-
&& usermod -aG sudo ${USERNAME} \
117-
&& echo "%sudo ALL=NOPASSWD:/usr/bin/apt-get update, /usr/bin/apt-get upgrade, /usr/bin/apt-get install, /usr/bin/apt-get remove" >> /etc/sudoers
55+
mkdir -p /build && \
56+
cmake .. -GNinja "-DCMAKE_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}" \
57+
-DCMAKE_INSTALL_PREFIX=/build/colmap && \
58+
ninja install -j1 && \
59+
cd ~
60+
61+
# Upgrade pip and install dependencies.
62+
# pip install torch==2.2.2 torchvision==0.17.2 --index-url https://download.pytorch.org/whl/cu118 && \
63+
RUN pip install --no-cache-dir --upgrade pip 'setuptools<70.0.0' && \
64+
pip install --no-cache-dir torch==2.1.2+cu118 torchvision==0.16.2+cu118 'numpy<2.0.0' --extra-index-url https://download.pytorch.org/whl/cu118 && \
65+
git clone --branch master --recursive https://github.com/cvg/Hierarchical-Localization.git /opt/hloc && \
66+
cd /opt/hloc && git checkout v1.4 && python3.10 -m pip install --no-cache-dir . && cd ~ && \
67+
TCNN_CUDA_ARCHITECTURES="${CUDA_ARCHITECTURES}" pip install --no-cache-dir "git+https://github.com/NVlabs/tiny-cuda-nn.git@b3473c81396fe927293bdfd5a6be32df8769927c#subdirectory=bindings/torch" && \
68+
pip install --no-cache-dir pycolmap==0.6.1 pyceres==2.1 omegaconf==2.3.0
69+
70+
# Install gsplat and nerfstudio.
71+
# NOTE: both are installed jointly in order to prevent docker cache with latest
72+
# gsplat version (we do not expliticly specify the commit hash).
73+
#
74+
# We set MAX_JOBS to reduce resource usage for GH actions:
75+
# - https://github.com/nerfstudio-project/gsplat/blob/db444b904976d6e01e79b736dd89a1070b0ee1d0/setup.py#L13-L23
76+
COPY --from=source /tmp/nerfstudio/ /tmp/nerfstudio
77+
RUN export TORCH_CUDA_ARCH_LIST="$(echo "$CUDA_ARCHITECTURES" | tr ';' '\n' | awk '$0 > 70 {print substr($0,1,1)"."substr($0,2)}' | tr '\n' ' ' | sed 's/ $//')" && \
78+
export MAX_JOBS=4 && \
79+
GSPLAT_VERSION="$(sed -n 's/.*gsplat==\s*\([^," '"'"']*\).*/\1/p' /tmp/nerfstudio/pyproject.toml)" && \
80+
pip install --no-cache-dir git+https://github.com/nerfstudio-project/gsplat.git@v${GSPLAT_VERSION} && \
81+
pip install --no-cache-dir /tmp/nerfstudio 'numpy<2.0.0' && \
82+
rm -rf /tmp/nerfstudio
83+
84+
# Fix permissions
85+
RUN chmod -R go=u /usr/local/lib/python3.10 && \
86+
chmod -R go=u /build
87+
88+
#
89+
# Docker runtime stage.
90+
#
91+
FROM nvidia/cuda:${NVIDIA_CUDA_VERSION}-runtime-ubuntu${UBUNTU_VERSION} as runtime
92+
ARG CUDA_ARCHITECTURES
93+
ARG NVIDIA_CUDA_VERSION
94+
ARG UBUNTU_VERSION
11895

119-
# Create workspace folder and change ownership to new user
120-
RUN mkdir /workspace && chown ${USER_UID}:${USER_GID} /workspace
121-
122-
# Switch to new user and workdir.
123-
USER ${USER_UID}
124-
WORKDIR /home/${USERNAME}
125-
126-
# Add local user binary folder to PATH variable.
127-
ENV PATH="${PATH}:/home/${USERNAME}/.local/bin"
128-
129-
# Upgrade pip and install packages.
130-
RUN python3.10 -m pip install --no-cache-dir --upgrade pip setuptools==69.5.1 pathtools promise pybind11 omegaconf
131-
132-
# Install pytorch and submodules
133-
# echo "${CUDA_VERSION}" | sed 's/.$//' | tr -d '.' -- CUDA_VERSION -> delete last digit -> delete all '.'
134-
RUN CUDA_VER=$(echo "${CUDA_VERSION}" | sed 's/.$//' | tr -d '.') && python3.10 -m pip install --no-cache-dir \
135-
torch==2.1.2+cu${CUDA_VER} \
136-
torchvision==0.16.2+cu${CUDA_VER} \
137-
--extra-index-url https://download.pytorch.org/whl/cu${CUDA_VER}
138-
139-
# Install tiny-cuda-nn (we need to set the target architectures as environment variable first).
140-
ENV TCNN_CUDA_ARCHITECTURES=${CUDA_ARCHITECTURES}
141-
RUN python3.10 -m pip install --no-cache-dir git+https://github.com/NVlabs/tiny-cuda-nn.git#subdirectory=bindings/torch
142-
143-
# Install pycolmap, required by hloc.
144-
RUN git clone --branch v0.4.0 --recursive https://github.com/colmap/pycolmap.git && \
145-
cd pycolmap && \
146-
python3.10 -m pip install --no-cache-dir . && \
147-
cd ..
148-
149-
# Install hloc 1.4 as alternative feature detector and matcher option for nerfstudio.
150-
RUN git clone --branch master --recursive https://github.com/cvg/Hierarchical-Localization.git && \
151-
cd Hierarchical-Localization && \
152-
git checkout v1.4 && \
153-
python3.10 -m pip install --no-cache-dir -e . && \
154-
cd ..
155-
156-
# Install pyceres from source
157-
RUN git clone --branch v1.0 --recursive https://github.com/cvg/pyceres.git && \
158-
cd pyceres && \
159-
python3.10 -m pip install --no-cache-dir -e . && \
160-
cd ..
161-
162-
# Install pixel perfect sfm.
163-
RUN git clone --recursive https://github.com/cvg/pixel-perfect-sfm.git && \
164-
cd pixel-perfect-sfm && \
165-
git reset --hard 40f7c1339328b2a0c7cf71f76623fb848e0c0357 && \
166-
git clean -df && \
167-
python3.10 -m pip install --no-cache-dir -e . && \
168-
cd ..
169-
170-
# Copy nerfstudio folder and give ownership to user.
171-
COPY --chown=${USER_UID}:${USER_GID} . /home/${USERNAME}/nerfstudio
172-
173-
# Install nerfstudio dependencies.
174-
RUN cd nerfstudio && \
175-
python3.10 -m pip install --no-cache-dir -e . && \
176-
cd ..
96+
LABEL org.opencontainers.image.source = "https://github.com/nerfstudio-project/nerfstudio"
97+
LABEL org.opencontainers.image.licenses = "Apache License 2.0"
98+
LABEL org.opencontainers.image.base.name="docker.io/library/nvidia/cuda:${NVIDIA_CUDA_VERSION}-devel-ubuntu${UBUNTU_VERSION}"
99+
LABEL org.opencontainers.image.documentation = "https://docs.nerf.studio/"
177100

178-
# Switch to workspace folder and install nerfstudio cli auto completion
179-
WORKDIR /workspace
180-
RUN ns-install-cli --mode install
101+
# Minimal dependencies to run COLMAP binary compiled in the builder stage.
102+
# Note: this reduces the size of the final image considerably, since all the
103+
# build dependencies are not needed.
104+
RUN apt-get update && \
105+
apt-get install -y --no-install-recommends --no-install-suggests \
106+
libboost-filesystem1.74.0 \
107+
libboost-program-options1.74.0 \
108+
libc6 \
109+
libceres2 \
110+
libfreeimage3 \
111+
libgcc-s1 \
112+
libgl1 \
113+
libglew2.2 \
114+
libgoogle-glog0v5 \
115+
libqt5core5a \
116+
libqt5gui5 \
117+
libqt5widgets5 \
118+
python3.10 \
119+
python3.10-dev \
120+
build-essential \
121+
python-is-python3 \
122+
ffmpeg
123+
124+
# Copy packages from builder stage.
125+
COPY --from=builder /build/colmap/ /usr/local/
126+
COPY --from=builder /usr/local/lib/python3.10/dist-packages/ /usr/local/lib/python3.10/dist-packages/
127+
COPY --from=builder /usr/local/bin/ns* /usr/local/bin/
128+
129+
# Install nerfstudio cli auto completion
130+
RUN /bin/bash -c 'ns-install-cli --mode install'
181131

182132
# Bash as default entrypoint.
183133
CMD /bin/bash -l
184-
# Force changing password on first container run
185-
# Change line above: CMD /bin/bash -l -> CMD /bin/bash -l -c passwd && /usr/bin/bash -l

0 commit comments

Comments
 (0)