Merge pull request #7 from BlazingDB/feature/google-cloud-storage

[REVIEW] Google Cloud Storage

Author: aucahuasi

CMakeLists.txt

@@ -5,111 +5,32 @@
 
 cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 
-project(blazingdb-dependencies C CXX CUDA)
+project(blazingdb-dependencies C CXX)
 
 set(CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake/" ${CMAKE_MODULE_PATH})
 message(STATUS "CMAKE_MODULE_PATH:" "${CMAKE_MODULE_PATH}")
 
-## rmm - git@github.com:rapidsai/rmm.git
-option(ENABLE_RMM "Add the rmm (rapidsai) library" OFF)
-
-## custrings - git@github.com:rapidsai/custrings.git
-option(ENABLE_CUSTRINGS "Add the custring (rapidsai) library" OFF)
-
-# Parse the COMPONENT_VERSION and the component download URLs
-include(DefineVersions)
-
-# NOTE First step
-include(ConfigureBlazingDBHostCompiler)
-
 # Include CMake modules
 include(FeatureSummary)
 include(CheckIncludeFiles)
 include(CheckLibraryExists)
 include(CTest)
 
-# Configure CUDA
-include(ConfigureCUDA)
-
 # NOTE Do not change the inclusion order
 # Configure dependencies
-include(ConfigureGoogleTest) # NOTE for unit tests
-include(ConfigureGoogleBenchmark) # NOTE for performance tests
-include(ConfigureBoost)
-include(ConfigureRapidJSON) # TODO remove when we use binary protocol with ucx
-include(ConfigureSimpleWebServer) # TODO remove when we use binary protocol with ucx
-include(ConfigureAWSSDK)
-include(ConfigureFlatBuffers)
-include(ConfigureLz4)
-include(ConfigureZSTD)
-include(ConfigureBrotli)
-include(ConfigureSnappy)
-include(ConfigureThrift)
-include(ConfigureArrow)
-include(ConfigureParquet)
-include(ConfigurePython3)
-
-include(ConfigureJitify) # NOTE cudf related
-
-if(ENABLE_RMM)
-    message(STATUS "Enable the rmm (rapidsai) library")
-    include(ConfigureRMM)
-else()
-    message(STATUS "Disable the rmm (rapidsai) library")
-endif()
-
-if(ENABLE_CUSTRINGS)
-    message(STATUS "Enable the custring (rapidsai) library")
-    include(ConfigureNVStrings)
-else()
-    message(STATUS "Disable the custring (rapidsai) library")
-endif()
-
-#find_package(Threads)
-
-# NOTE Optional build: cudf
-#include(ConfigureLibGDF)
-
-# NOTE Optional build: BlazingSQL stack
-#message(FATAL_ERROR "AAAAAAAAAAAAAAAAAAAA@")
-#include(ConfigureBlazingDBIO)
+# TODO percy add support for aws too latr
+# include(ConfigureAWSSDK)
+include(ConfigureGoogleCloudCPP)
 
 # BEGIN MAIN #
 # NOTE use USE_SOURCE_PERMISSIONS to preserve executable file permisions (e.g. flatc, etc)
-install(DIRECTORY ${GTEST_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${GBENCHMARK_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${BOOST_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${RAPIDJSON_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS) # TODO remove when we use binary protocol with ucx
-install(DIRECTORY ${SIMPLEWEBSERVER_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS) # TODO remove when we use binary protocol with ucx
-install(DIRECTORY ${AWS_SDK_CPP_ROOT} DESTINATION build/aws-sdk-cpp USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${FLATBUFFERS_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${LZ4_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${ZSTD_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${BROTLI_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${SNAPPY_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${THRIFT_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${ARROW_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-install(DIRECTORY ${PYTHON3_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-
-## rmm - git@github.com:rapidsai/rmm.git
-if(ENABLE_RMM)
-    install(DIRECTORY ${RMM_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-endif()
-
-## custrings - git@github.com:rapidsai/custrings.git
-if(ENABLE_CUSTRINGS)
-    install(DIRECTORY ${NVSTRINGS_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
-endif()
-
-install(FILES ${JITIFY_ROOT}/jitify.hpp DESTINATION include) # NOTE cudf related
+install(DIRECTORY ${AWS_SDK_CPP_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
+install(DIRECTORY ${GOOGLE_CLOUD_CPP_ROOT} DESTINATION . USE_SOURCE_PERMISSIONS)
 
 # Install the cmake modules to be used by the users/projects
 install(DIRECTORY ${CMAKE_SOURCE_DIR}/cmake/ DESTINATION lib/cmake)
 
-#unnecessary becouse arrow installation step was already performed
-#install(DIRECTORY ${PARQUET_ROOT} DESTINATION .)
-
 # Print the project summary
 feature_summary(WHAT ALL INCLUDE_QUIET_PACKAGES FATAL_ON_MISSING_REQUIRED_PACKAGES)
 
-#END MAIN #
+# END MAIN #

README.md

@@ -1,82 +1,36 @@
-# BlazingSQL dependencies
+# BlazingSQL Toolchain
+
+This project contains software development tools and dependencies to develop and 
+build BlazingSQL using Anaconda environments.
 
 # Requirements
 
 ## Compiler
 
-- g++-5.4
-
-## Builders
-
-- Make
-
-If you want to use Ninja (which is optional) on Ubuntu 16.04 then just run:
-
-```bash
-apt-get -y install ninja-build
-```
+- conda
 
 ## CMake
 
 We need at least CMake version 3.12+. Follow these commands to install CMake:
 
-```bash
-wget https://github.com/Kitware/CMake/releases/download/v3.14.1/cmake-3.14.1-Linux-x86_64.tar.gz
-tar xvf cmake-3.14.1-Linux-x86_64.tar.gz 
-# Add cmake-3.14.1-Linux-x86_64/bin to your PATH
-cmake --version
-cmake version 3.14.1
-
-CMake suite maintained and supported by Kitware (kitware.com/cmake).
-```
-
-**NOTE:**
-**Never build CMake**. Use **always** the **official binary** releases!!!
+**Never build CMake**. Use **always** the **official binary** from conda releases!!!
 
 ## System Requirements
 
 And also make sure to install these system requirements:
-```bash
-# Install common dev tools
-apt-get install -y build-essential ssh wget curl git
-
-# Add autotools suite to build some Apache Parquet dependencies
-apt-get -y install libtool automake autoconf
-
-# Install Boost regex dependency
-apt-get install -y libicu-dev
-
-# Install AWS C++ SDK dependencies
-apt-get install -y --no-install-recommends libcurl4-openssl-dev libssl-dev uuid-dev zlib1g-dev
-
-# Install Apache Arrow / Thrift dependencies
-apt-get install -y libssl-dev libtool bison flex pkg-config
-```
-
-# Build
 
 ```bash
-cd blazingdb-dependencies
-mkdir -p build
-cd build
-CUDACXX=/usr/local/cuda-9.2/bin/nvcc cmake -DCMAKE_INSTALL_PREFIX=/foo/blazingsql/dependencies/ ..
-make -j8 install
-```
-
-**NOTE:**
-If you want to build the dependencies using the old C++ ABI, run the cmake command like this:
+# Add classic tools and autotools suite
+apt-get -y install wget libtool automake autoconf pkg-config
 
-```bash
-CUDACXX=/usr/local/cuda-9.2/bin/nvcc cmake -DCXX_OLD_ABI=ON -DCMAKE_INSTALL_PREFIX=/foo/blazingsql/dependencies/ ..
+# Install system dependencies
+#apt-get install -y --no-install-recommends uuid-dev zlib1g-dev
 ```
 
-## Optional Packages
-All the packages are installed by default.
+# Build
 
-In case the custring (NVString) package is not required, add the cmake parameter 'ENABLE_CUSTRINGS' as follow:
 ```bash
-cmake -DENABLE_CUSTRINGS=OFF -DCMAKE_INSTALL_PREFIX=/folder/dependencies/ ..
+CURRENT_CONDA_PREFIX=$CONDA_PREFIX conda build --label "main" -c conda-forge --python=3.7 --output-folder /some/directory/
 ```
 
-# List of Packages
-* custring (NVString) - `git@github.com:rapidsai/custrings.git`
+For more details, see the conda-build process.

ci/cpu/build.sh

@@ -0,0 +1,50 @@
+#!/bin/bash
+# Copyright (c) 2019, BLAZINGSQL.
+######################################
+# cuDF CPU conda build script for CI #
+######################################
+set -e
+
+# Logger function for build status output
+function logger() {
+  echo -e "\n>>>> $@\n"
+}
+
+# Set home to the job's workspace
+export HOME=$WORKSPACE
+
+# Switch to project root; also root of repo checkout
+cd $WORKSPACE
+
+# Get latest tag and number of commits since tag
+export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags`
+export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count`
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+logger "Get env..."
+env
+
+logger "Check versions..."
+python --version
+gcc --version
+g++ --version
+conda list
+
+# FIX Added to deal with Anancoda SSL verification issues during conda builds
+conda config --set ssl_verify False
+
+logger "Install dependencies..."
+conda install -y conda-build anaconda-client
+
+################################################################################
+# BUILD - Conda package builds
+################################################################################
+
+logger "Build conda pkg for communication..."
+source ci/cpu/toolchain/conda-build.sh
+
+logger "Upload conda pkg for communication..."
+source ci/cpu/upload_anaconda.sh

ci/cpu/toolchain/conda-build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+conda build -c conda-forge -c defaults --python=$PYTHON conda/recipes/blazingdb-toolchain/

ci/cpu/upload_anaconda.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+export TAR_FILE=`conda build conda/recipes/blazingdb-toolchain --python=$PYTHON --output`
+
+LABEL_OPTION="--label main"
+echo "LABEL_OPTION=${LABEL_OPTION}"
+
+if [ -z "$MY_UPLOAD_KEY" ]; then
+    echo "No upload key"
+    return 0
+fi
+
+test -e ${TAR_FILE}
+echo "Upload communication"
+echo ${TAR_FILE}
+anaconda -t ${MY_UPLOAD_KEY} upload -u blazingsql ${LABEL_OPTION} --force ${TAR_FILE}