Merge pull request #565 from nicolasvasilache/pr/llvm-trunk

Move to trunk LLVM

Author: nicolasvasilache

.circleci/config.yml

@@ -9,11 +9,11 @@ jobs:
     steps:
       - checkout
       - run:
-          name: conda_tapir_halide
+          name: conda_llvm_halide
           command: |
             . /opt/conda/anaconda/bin/activate
             source activate tc_build
-            conda install -y -c nicolasvasilache llvm-tapir50 halide
+            conda install -y -c nicolasvasilache llvm-trunk halide
 
       - run:
           name: check_formatting

.github/ISSUE_TEMPLATE.md

@@ -16,7 +16,7 @@ When submitting a bug report, please include the following information (where re
 - GCC/GXX version (if compiling from source):
 - LLVM/Tapir git hash used (if compiling from source):
 
-  To get the hash, run: `$HOME/clang+llvm-tapir5.0/bin/clang --version`
+  To get the hash, run: `$CONDA_PREFIX/bin/clang --version`
 
 - Commit hash of our repo and submodules (if compiling from source):
 

.jenkins/build.sh

@@ -60,7 +60,7 @@ cd /var/lib/jenkins/workspace
 git submodule update --init --recursive
 
 source activate tc_build
-conda install -y -c nicolasvasilache llvm-tapir50 halide
+conda install -y -c nicolasvasilache llvm-trunk halide
 conda install -y -c conda-forge eigen
 conda install -y -c nicolasvasilache caffe2
 

conda_recipes/README.md

@@ -17,7 +17,7 @@ nvidia-docker run --rm -i -t tc-cuda9.0-cudnn7.1-ubuntu16.04-devel
 
 We are ready to build conda package for TC.
 To simplify the build process we ship TC dependencies as conda packages.
-We need to build packages for `llvm-tapir50`, `Halide`, `Caffe2` (optional) and finally `Tensor Comprehensions`.
+We need to build packages for `llvm-trunk`, `Halide`, `Caffe2` (optional) and finally `Tensor Comprehensions`.
 
 For building each package, we need to specify a `build version`, `build number` and
 `git hash`. This information is used to build each package.

conda_recipes/conda_build_tc.sh

@@ -28,27 +28,31 @@ time conda build -c $ANACONDA_USER --python 3.6 caffe2
 echo "Caffe2 packaged Successfully"
 
 ###############################################################################
-# LLVM_TAPIR settings
-LLVM_TAPIR_BUILD_VERSION="1.0.0"
-LLVM_TAPIR_BUILD_NUMBER=1
-LLVM_TAPIR_GIT_HASH="1482504e234a65bffc8c54de8de9fc877822345d"
+# LLVM_TRUNK settings
+LLVM_TRUNK_BUILD_VERSION="1.0.0"
+LLVM_TRUNK_BUILD_NUMBER=1
+LLVM_TRUNK_SOURCE_DIR=$(mktemp -d /tmp/d.XXXXXX)
+trap 'rm -rf "${LLVM_TRUNK_SOURCE_DIR}"' EXIT
 
-echo "Building llvm-tapir50"
-echo "LLVM_TAPIR_BUILD_VERSION: $LLVM_TAPIR_BUILD_VERSION LLVM_TAPIR_BUILD_NUMBER: ${LLVM_TAPIR_BUILD_NUMBER}"
+svn co http://llvm.org/svn/llvm-project/llvm/trunk ${LLVM_TRUNK_SOURCE_DIR}
+svn co http://llvm.org/svn/llvm-project/cfe/trunk ${LLVM_TRUNK_SOURCE_DIR}/tools/clang
 
-export LLVM_TAPIR_BUILD_VERSION=$LLVM_TAPIR_BUILD_VERSION
-export LLVM_TAPIR_BUILD_NUMBER=$LLVM_TAPIR_BUILD_NUMBER
-export LLVM_TAPIR_GIT_HASH=$LLVM_TAPIR_GIT_HASH
+echo "Building llvm-trunk"
+echo "LLVM_TRUNK_BUILD_VERSION: $LLVM_TRUNK_BUILD_VERSION LLVM_TRUNK_BUILD_NUMBER: ${LLVM_TRUNK_BUILD_NUMBER}"
 
-time conda build -c $ANACONDA_USER --python 3.6 llvm-tapir50
+export LLVM_TRUNK_BUILD_VERSION=$LLVM_TRUNK_BUILD_VERSION
+export LLVM_TRUNK_BUILD_NUMBER=$LLVM_TRUNK_BUILD_NUMBER
+export LLVM_TRUNK_SOURCE_DIR=$LLVM_TRUNK_SOURCE_DIR
 
-echo "llvm-tapir50 packaged Successfully"
+time conda build -c $ANACONDA_USER --python 3.6 llvm-trunk
+
+echo "llvm-trunk packaged Successfully"
 
 ##############################################################################
 # Halide settings
 HALIDE_BUILD_VERSION="1.0.0"
-HALIDE_BUILD_NUMBER=0
-HALIDE_GIT_HASH="35be67b3a3e4c4461f79949109ff35c54cf307de"
+HALIDE_BUILD_NUMBER=1
+HALIDE_GIT_HASH="0b29cacf636852933892bbaa61dd2050c8dcaff2"
 
 echo "Packaging HALIDE ==> HALIDE_BUILD_VERSION: ${HALIDE_BUILD_VERSION} HALIDE_BUILD_NUMBER: ${HALIDE_BUILD_NUMBER}"
 
@@ -75,4 +79,3 @@ echo "HALIDE packaged Successfully"
 #time conda build -c $ANACONDA_USER --python 3.6 tensor_comprehensions
 #
 #echo "Tensor Comprehensions packaged Successfully"
-#

conda_recipes/halide/meta.yaml

@@ -8,10 +8,10 @@ source:
 
 requirements:
   build:
-    - llvm-tapir50==1.0.0
+    - llvm-trunk==1.0.0
     - cmake
   run:
-    - llvm-tapir50==1.0.0
+    - llvm-trunk==1.0.0
     - cmake
 
 build:

conda_recipes/llvm-trunk/build.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -e
+
+# code gets checked out at below:
+# $SOURCE_DIR ===> <anaconda_root>/conda-bld/llvm-trunk_<timestamp>/work
+#
+# build directory gets created at $SOURCE_DIR/build
+#
+# CONDA environment for debugging:
+# cd <anaconda_root>/conda-bld/llvm-trunk_<timestamp>
+# source activate ./_h_env_......    # long placeholders
+#
+# $CONDA_PREFIX and $PREFIX are set to the same value i.e. the environment value
+#
+# Installation happens in the $PREFIX which is the environment and rpath is set
+# to that
+#
+# For tests, a new environment _test_env_.... is created
+# During the tests, you will see that the llvm-trunk package gets checked out
+# NOTE: once the build finished, the packaging and unpackaging step takes long
+# and might seem like it's stuck but it's not.
+
+CMAKE_VERSION=${CMAKE_VERSION:="`which cmake3 || which cmake`"}
+VERBOSE=${VERBOSE:=0}
+
+echo "CONDA_PREFIX: ${CONDA_PREFIX}"
+echo "PREFIX: ${PREFIX}"
+
+echo "Building llvm-trunk conda package"
+
+echo "Clean up existing build packages if any"
+rm -rf llvm_build || true
+
+mkdir -p llvm_build
+cd llvm_build
+
+echo "Configuring llvm-trunk"
+VERBOSE=${VERBOSE}  ${CMAKE_VERSION} \
+    -DCMAKE_INSTALL_PREFIX=${PREFIX} \
+    -DCOMPILER_RT_BUILD_CILKTOOLS=OFF \
+    -DLLVM_ENABLE_CXX1Y=ON \
+    -DLLVM_ENABLE_TERMINFO=OFF \
+    -DLLVM_BUILD_TESTS=OFF \
+    -DLLVM_ENABLE_ASSERTIONS=ON \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_BUILD_LLVM_DYLIB=ON  \
+    -DLLVM_TARGETS_TO_BUILD="X86;NVPTX" \
+    -DLLVM_ENABLE_EH=ON \
+    -DLLVM_ENABLE_OCAMLDOC=OFF \
+    -DLLVM_INSTALL_OCAMLDOC_HTML_DIR=/tmp \
+    -DLLVM_OCAML_INSTALL_PATH=/tmp \
+    -DLLVM_ENABLE_RTTI=ON .. || exit 1
+
+echo "Installing llvm-trunk"
+VERBOSE=${VERBOSE}  make -j"$(nproc)" -s || exit 1
+
+VERBOSE=${VERBOSE}  make install -j"$(nproc)" -s || exit 1
+
+echo SUCCESS || echo FAILURE
+
+echo "Successfully built llvm-trunk conda package"

conda_recipes/llvm-trunk/meta.yaml

@@ -0,0 +1,33 @@
+package:
+  name: llvm-trunk
+  version: "{{ environ.get('LLVM_TRUNK_BUILD_VERSION') }}"
+
+source:
+  path: "{{ environ.get('LLVM_TRUNK_SOURCE_DIR') }}"
+
+requirements:
+  build:
+    - cmake
+  run:
+    - cmake
+
+build:
+  number: {{ environ.get('LLVM_TRUNK_BUILD_NUMBER') }}
+  skip: True   # [win]
+
+test:
+  commands:
+    - $PREFIX/bin/llvm-config --libs
+    - $PREFIX/bin/llc -version
+    - test -f $PREFIX/include/llvm/Pass.h
+    - test -f $PREFIX/lib/libLLVMCore.a
+
+    - $PREFIX/bin/clang --version
+    - $PREFIX/bin/clang++ --version
+    - $PREFIX/bin/clang-check --version
+    - test -f $PREFIX/include/clang/Lex/Lexer.h
+    - test -f $PREFIX/lib/libclangBasic.a
+
+extra:
+  recipe-maintainers:
+    - nicolasvasilache

conda_recipes/tensor_comprehensions/meta.yaml

@@ -8,7 +8,7 @@ source:
 
 requirements:
   build:
-    - llvm-tapir50==1.0.0
+    - llvm-trunk==1.0.0
     - halide==1.0.0
     - caffe2==1.0.0
     - pytorch==0.4.0
@@ -17,7 +17,7 @@ requirements:
     - setuptools
 
   run:
-    - llvm-tapir50==1.0.0
+    - llvm-trunk==1.0.0
     - halide==1.0.0
     - caffe2==1.0.0
     - pytorch==0.4.0

docs/source/installation.rst

@@ -66,7 +66,7 @@ Create a new environment in which TC will be built and install core dependencies
        conda create -y --name tc_build python=3.6
        conda activate tc_build
        conda install -y pyyaml mkl-include pytest
-       conda install -y -c nicolasvasilache llvm-tapir50 halide
+       conda install -y -c nicolasvasilache llvm-trunk halide
 
 Then install the PyTorch version that corresponds to your system binaries
 (e.g. cuda 9.0):