Merge pull request #122 from facebookresearch/docker-jenkins

Refactor dockerfiles and install jenkins user

Author: prigoyal

.jenkins/build.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -ex
+
+source /etc/lsb-release
+
+# condition: if 14.04 and conda, conda install pytorch and build
+# condition: if 16.04 and conda, conda install pytorch and build
+# condition: if any and non-conda, simply build TC from scratch
+
+if [[ "$DISTRIB_RELEASE" == 14.04 ]]; then
+  if which conda &> /dev/null; then
+    echo "Building TC in conda env"
+    conda install -y -c pytorch pytorch
+    conda install -y pyyaml
+    WITH_PYTHON_C2=OFF CLANG_PREFIX=/usr/local/clang+llvm-tapir5.0 BUILD_TYPE=Release ./build.sh --all
+  else
+    echo "Building TC in non-conda env"
+    WITH_PYTHON_C2=OFF CLANG_PREFIX=/usr/local/clang+llvm-tapir5.0 BUILD_TYPE=Release ./build.sh --all
+  fi
+fi
+
+if [[ "$DISTRIB_RELEASE" == 16.04 ]]; then
+  if which conda &> /dev/null; then
+    echo "Building TC in conda env"
+    conda install -y pytorch cuda90 -c pytorch
+    conda install -y pyyaml
+    WITH_PYTHON_C2=OFF CLANG_PREFIX=/usr/local/clang+llvm-tapir5.0 BUILD_TYPE=Release ./build.sh --all
+  else
+    echo "Building TC in non-conda env"
+    WITH_PYTHON_C2=OFF CLANG_PREFIX=/usr/local/clang+llvm-tapir5.0 BUILD_TYPE=Release ./build.sh --all
+  fi
+fi

.jenkins/run_test.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -ex
+
+source /etc/lsb-release
+
+# condition: if 14.04 and conda, also run python tests
+# condition: if 16.04 and conda, run only python tests
+# condition: if any and non-conda, run test.sh only
+
+if [[ "$DISTRIB_RELEASE" == 14.04 ]]; then
+  echo "Running TC backend tests"
+  ./test.sh
+  if which conda &> /dev/null; then
+    source activate
+    echo "Running TC PyTorch tests"
+    ./test_python/run_test.sh
+  fi
+fi
+
+if [[ "$DISTRIB_RELEASE" == 16.04 ]]; then
+  if which conda &> /dev/null; then
+    echo "Running TC PyTorch tests"
+    source activate
+    ./test_python/run_test.sh
+  else
+    echo "Running TC backend tests"
+    ./test.sh
+  fi
+fi

docker/README.md

@@ -1,9 +1,36 @@
 # How to build Tensor Comprehensions docker images
 
 Tensor Comprehensions supports gcc4.8, gcc5, cuda8-cudnn6, cuda9-cudnn7, Xenial, Trusty
-and building in Conda vs no conda environment. Below are the steps you can follow
-to build the images for various combinations. The docker image is built in a ladder
-manner so as to reuse the images. Steps:
+and building in Conda/ Non-conda environment. Below are the steps you can follow
+to build the images for various combinations. This directory contains everything needed to build the Docker images that are used in our CI as well. The `Dockerfile`s are parameterized to build the image based on the build environment. The different configurations are identified by a freeform string that we call a build environment. This string is persisted in each image as the BUILD_ENVIRONMENT environment variable.
+
+## Supported Build environments
+
+Non-conda env and no python installs:
+
+`linux-trusty-gcc4.8-cuda8-cudnn7-py3`
+
+`linux-xenial-gcc5-cuda9-cudnn7-py3`
+
+Conda env with PyTorch conda install:
+
+`linux-trusty-gcc4.8-cuda8-cudnn7-py3-conda` [Run both C++ and Python tests]
+
+`linux-xenial-gcc5-cuda9-cudnn7-py3-conda` [Run only Python tests for now]
+
+See `docker_build.sh` for a full list of terms that are extracted from the build environment into parameters for the image build.
+
+## Contents
+
+`docker_build.sh` -- dispatch script to launch all builds
+
+`common` -- scripts used to execute individual Docker build stages
+
+`ubuntu-cuda` -- Dockerfile for Ubuntu image with CUDA support in non-conda env
+
+`ubuntu-cuda-conda` -- Dockerfile for Ubuntu image with CUDA support in non-conda env
+
+## Steps to build docker image
 
 **NOTE**: You need to have docker installed on your system. Follow the instructions
 on docker website to install it, if you don't have docker already.
@@ -34,20 +61,23 @@ need to build images for some other combination, the steps are still valid, just
 make sure to change the paths below to fit your case.
 
 ```Shell
-docker build -t tensorcomprehensions/linux-trusty-gcc4.8-cuda8-cudnn6-py3-conda .
+image=linux-trusty-gcc4.8-cuda8-cudnn7-py3-conda ./docker_build.sh
 ```
 
 This will build the image for the above permutation and then we can test this image
 
 6. Test the image
 
+run `docker images` and get the `IMAGE_ID` for the image we just built.
+
 ```Shell
-docker run --runtime=nvidia -i -t tensorcomprehensions/linux-trusty-gcc4.8-cuda8-cudnn6-py3-conda:2
+docker run --runtime=nvidia -i -t ${IMAGE_ID}
 # now clone the TC repo
 cd $HOME && git clone https://github.com/facebookresearch/TensorComprehensions.git --recursive && cd TensorComprehensions
 # build TC
+conda install -y -c pytorch pytorch
 WITH_PYTHON_C2=OFF CLANG_PREFIX=/usr/local/clang+llvm-tapir5.0 ./build.sh --all
 # Test the TC build is fine
-./test_cpu.sh
 ./test.sh
+./test_python/run_test.sh
 ```

docker/linux-trusty-gcc4.8-cuda8-cudnn6-py3-conda/Dockerfile renamed to docker/archive/linux-trusty-gcc4.8-cuda8-cudnn6-py3-conda/Dockerfile

@@ -41,7 +41,7 @@ RUN ln -s /usr/local/cuda/targets/x86_64-linux/lib/stubs/libcuda.so /usr/local/c
 ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:/usr/local/cuda/targets/x86_64-linux/lib/stubs/:$LD_LIBRARY_PATH
 ENV PATH /usr/local/bin:/usr/local/cuda/bin:$PATH
 
-# Conda setup
+# Anaconda3 setup
 WORKDIR /conda-install
 RUN echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh &&\
     wget --quiet https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh -O anaconda.sh && \

docker/linux-trusty-gcc4.8-cuda8-cudnn6-py3/Dockerfile renamed to docker/archive/linux-trusty-gcc4.8-cuda8-cudnn6-py3/Dockerfile

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -ex
+
+# Mirror jenkins user in container
+echo "jenkins:x:$JENKINS_UID:$JENKINS_GID::/var/lib/jenkins:" >> /etc/passwd
+echo "jenkins:x:$JENKINS_GID:" >> /etc/group
+
+# Create $HOME
+mkdir -p /var/lib/jenkins
+chown jenkins:jenkins /var/lib/jenkins
+mkdir -p /var/lib/jenkins/.ccache
+chown jenkins:jenkins /var/lib/jenkins/.ccache
+
+# Allow writing to /usr/local (for make install)
+chown jenkins:jenkins /usr/local
+
+# Allow sudo
+echo 'jenkins ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/jenkins

docker/linux-xenial-gcc5-cuda9-cudnn7-py3-conda/Dockerfile renamed to docker/archive/linux-xenial-gcc5-cuda9-cudnn7-py3-conda/Dockerfile

@@ -0,0 +1,85 @@
+#!/bin/bash
+
+set -ex
+
+# Install common dependencies
+apt-get update
+apt-get install -y --no-install-recommends \
+  curl \
+  make \
+  git \
+  ssh \
+  realpath \
+  wget \
+  unzip \
+  vim \
+  automake \
+  libtool \
+  valgrind \
+  subversion \
+  libgtest-dev \
+  libz-dev \
+  libgmp3-dev \
+  libyaml-dev \
+  libgoogle-glog-dev \
+  ca-certificates \
+  software-properties-common \
+  build-essential
+
+# setup gcc
+add-apt-repository ppa:ubuntu-toolchain-r/test
+apt-get update
+apt-get install -y --no-install-recommends libcilkrts5 gcc-$GCC_VERSION g++-$GCC_VERSION
+update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-$GCC_VERSION 50
+update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-$GCC_VERSION 50
+
+export CC=/usr/bin/gcc
+export CXX=/usr/bin/g++
+
+# install cmake - this unifies trusty/xenial cmake version > 3.4.3
+[ -n "$CMAKE_VERSION" ]
+
+# Turn 3.6.3 into v3.6
+path=$(echo "${CMAKE_VERSION}" | sed -e 's/\([0-9].[0-9]\+\).*/v\1/')
+file="cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz"
+# Download and install specific CMake version in /usr/local
+pushd /tmp
+curl -Os "https://cmake.org/files/${path}/${file}"
+tar -C /usr/local --strip-components 1 --no-same-owner -zxf cmake-*.tar.gz
+rm -f cmake-*.tar.gz
+popd
+
+
+# LLVM+Clang-Tapir5.0
+export LLVM_SOURCES=/tmp/llvm_sources-tapir5.0
+mkdir -p $LLVM_SOURCES
+pushd $LLVM_SOURCES
+
+export CLANG_PREFIX=/usr/local/clang+llvm-tapir5.0
+export CMAKE_VERSION=cmake
+
+git clone --recursive https://github.com/wsmoses/Tapir-LLVM llvm && \
+mkdir -p ${LLVM_SOURCES}/llvm_build && cd ${LLVM_SOURCES}/llvm_build && \
+${CMAKE_VERSION} -DLLVM_ENABLE_EH=ON \
+  -DLLVM_ENABLE_OCAMLDOC=OFF \
+  -DLLVM_INSTALL_OCAMLDOC_HTML_DIR=/tmp \
+  -DLLVM_OCAML_INSTALL_PATH=/tmp \
+  -DCMAKE_INSTALL_PREFIX=${CLANG_PREFIX} \
+  -DLLVM_TARGETS_TO_BUILD=X86 \
+  -DCOMPILER_RT_BUILD_CILKTOOLS=OFF \
+  -DLLVM_ENABLE_CXX1Y=ON \
+  -DLLVM_ENABLE_TERMINFO=OFF \
+  -DLLVM_BUILD_TESTS=OFF \
+  -DLLVM_ENABLE_ASSERTIONS=ON \
+  -DCMAKE_BUILD_TYPE=Release \
+  -DLLVM_BUILD_LLVM_DYLIB=ON  \
+  -DLLVM_ENABLE_RTTI=ON ../llvm/
+
+make -j"$(nproc)" -s && make install -j"$(nproc)" -s
+
+popd
+rm -Rf ${LLVM_SOURCES}
+
+# Cleanup package manager
+apt-get autoclean && apt-get clean
+rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*

docker/linux-xenial-gcc5-cuda9-cudnn7-py3/Dockerfile renamed to docker/archive/linux-xenial-gcc5-cuda9-cudnn7-py3/Dockerfile

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -ex
+
+export TEMP_INSTALL=/tmp/conda-install
+mkdir -p $TEMP_INSTALL
+
+pushd $TEMP_INSTALL
+
+echo 'export PATH=/opt/conda/bin:$PATH' > /etc/profile.d/conda.sh
+wget --quiet https://repo.continuum.io/archive/Anaconda3-5.0.1-Linux-x86_64.sh -O anaconda.sh
+chmod +x anaconda.sh
+/bin/bash ./anaconda.sh -b -p /opt/conda
+
+rm anaconda.sh
+
+popd
+
+export PATH=/opt/conda/bin:$PATH
+
+/opt/conda/bin/conda install numpy decorator six future cmake pyyaml
+
+which conda
+conda --version
+which python
+python --version
+
+# Cleanup package manager
+apt-get autoclean && apt-get clean
+rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*