Skip to content

Commit d586cbe

Browse files
tkiliastkilias
authored
Add new flavor python3-ds-cuda-preview (#40)
This commit adds a new flavor which enables the usage of CUDA CPUs in the Container. This is currently specifically meant for Tensorflow, but will be extended to other libraries. The flavor installs the CUDA Driver Version 410.104 without the Kernel module and the CUDA SDK Version 10 into the Container. These versions are specifically chosen for Tensorflow for GPUs in Version 1.13.1, because Tensorflow installed with pip depends on specific version of libcuda.so. To use Tensorflow with GPUs the Host system needs the complete CUDA Driver with the same version installed. NOTE: Currently, Travis CI is not able to upload the pre-packaged python3-ds-cuda-preview container to Github, because it fails with the following Error: travis-ci/travis-ci#8553 Further, minor changes: - bugfixes for the exaslct starter script on some installations of python3 - bugfix of the travis.yaml which now pushes all images to the build caches during deployment - added PyYAML to all python3-ds-* flavors
1 parent 369ee61 commit d586cbe

File tree

31 files changed

+640
-13
lines changed

31 files changed

+640
-13
lines changed

‎.travis.yml

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,9 @@ jobs:
7272
- name: "fancyr-EXASOL-6.1.0"
7373
script:
7474
- "travis/build_language_deps_and_build_deps.sh --flavor-path flavors/fancyr-EXASOL-6.1.0"
75+
- name: "python3-ds-cuda-preview-EXASOL-6.1.0"
76+
script:
77+
- "travis/build_language_deps_and_build_deps.sh --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0"
7578
##################################################################################################
7679
################################# Build --goal flavor_base_deps ##################################
7780
##################################################################################################
@@ -94,6 +97,9 @@ jobs:
9497
- name: "python3-ds-EXASOL-6.1.0 --goal flavor_base_deps"
9598
script:
9699
- "travis/build_flavor_base_deps.sh --flavor-path flavors/python3-ds-EXASOL-6.1.0"
100+
- name: "python3-ds-cuda-preview-EXASOL-6.1.0 --goal flavor_base_deps"
101+
script:
102+
- "travis/build_flavor_base_deps.sh --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0"
97103
##################################################################################################
98104
##################################### Build --goal build_run #####################################
99105
##################################################################################################
@@ -116,6 +122,9 @@ jobs:
116122
- name: "fancyr-EXASOL-6.1.0 --goal build_run"
117123
script:
118124
- "travis/build_build_run.sh --flavor-path flavors/fancyr-EXASOL-6.1.0"
125+
- name: "python3-ds-cuda-preview-EXASOL-6.1.0 --goal build_run"
126+
script:
127+
- "travis/build_build_run.sh --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0"
119128
##################################################################################################
120129
###################################### Build --goal release ######################################
121130
##################################################################################################
@@ -138,6 +147,9 @@ jobs:
138147
- name: "python3-ds-EXASOL-6.1.0"
139148
script:
140149
- "travis/build_release.sh --flavor-path flavors/python3-ds-EXASOL-6.1.0"
150+
- name: "python3-ds-cuda-preview-EXASOL-6.1.0"
151+
script:
152+
- "travis/build_release.sh --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0"
141153
##################################################################################################
142154
################################ Run DB Test standard-EXASOL-6.0.0 ###############################
143155
##################################################################################################
@@ -213,6 +225,19 @@ jobs:
213225
--test-folder=python3
214226
--test-folder=python3-ds-flavor"
215227
##################################################################################################
228+
############################### Run DB Test python3-ds-cuda-preview-EXASOL-6.1.0 ##############################
229+
##################################################################################################
230+
- stage: "Run DB Test"
231+
name: "python3-ds-cuda-preview-EXASOL-6.1.0 --generic-language-test=python3"
232+
script:
233+
- "travis/run_db_test.sh --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0
234+
--generic-language-test=python3"
235+
- name: "python3-ds-cuda-preview-EXASOL-6.1.0 --test-folder=python3 --test-folder=python3-ds-flavor"
236+
script:
237+
- "travis/run_db_test.sh --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0
238+
--test-folder=python3
239+
--test-folder=python3-ds-flavor"
240+
##################################################################################################
216241
############################### Run DB Test fancyr-EXASOL-6.0.0 ##################################
217242
##################################################################################################
218243
- stage: "Run DB Test"
@@ -252,6 +277,9 @@ jobs:
252277
- name: "python3-ds-EXASOL-6.1.0"
253278
script:
254279
- "travis/deploy_images_for_commit.sh --flavor-path flavors/python3-ds-EXASOL-6.1.0"
280+
- name: "python3-ds-cuda-preview-EXASOL-6.1.0"
281+
script:
282+
- "travis/deploy_images_for_commit.sh --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0"
255283
##################################################################################################
256284
############################## Deploy Images to public build cache ###############################
257285
##################################################################################################
@@ -274,6 +302,9 @@ jobs:
274302
- name: "python3-ds-EXASOL-6.1.0"
275303
script:
276304
- "travis/deploy_images_to_public.sh --flavor-path flavors/python3-ds-EXASOL-6.1.0"
305+
- name: "python3-ds-cuda-preview-EXASOL-6.1.0"
306+
script:
307+
- "travis/deploy_images_to_public.sh --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0"
277308
##################################################################################################
278309
################################### Deploy exported Container ####################################
279310
##################################################################################################
@@ -287,6 +318,7 @@ jobs:
287318
--flavor-path flavors/fancyr-EXASOL-6.1.0
288319
--flavor-path flavors/python3-ds-EXASOL-6.0.0
289320
--flavor-path flavors/python3-ds-EXASOL-6.1.0"
321+
# --flavor-path flavors/python3-ds-cuda-preview-EXASOL-6.1.0" TODO container to large for upload https://github.com/travis-ci/travis-ci/issues/8553
290322
deploy:
291323
provider: releases
292324
api_key: "$GH_TOKEN"

‎exaslct

Lines changed: 22 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,20 @@ run () {
2828
$PIPENV_BIN run python3 exaslct_src/exaslct.py $COMMAND_LINE_ARGS
2929
}
3030

31+
find_pip_bin () {
32+
if python3 -m pip list &> /dev/null
33+
then
34+
PIP_BIN="python3 -m pip "
35+
else
36+
echo "ERROR: cant find pip"
37+
exit 1
38+
fi
39+
}
40+
3141
find_and_run_via_pip () {
32-
local PIPENV_LOCATION=$(pip show pipenv | grep 'Location:' | cut -f 2 -d " ")
33-
local PIPENV_BIN_IN_LOCATION=$(pip show -f pipenv | grep 'bin/pipenv$' | awk '{$1=$1};1')
42+
find_pip_bin
43+
local PIPENV_LOCATION=$($PIP_BIN show pipenv | grep 'Location:' | cut -f 2 -d " ")
44+
local PIPENV_BIN_IN_LOCATION=$($PIP_BIN show -f pipenv | grep 'bin/pipenv$' | awk '{$1=$1};1')
3445
PIPENV_BIN=$(command -v "$PIPENV_LOCATION/$PIPENV_BIN_IN_LOCATION")
3546
if [ -n "$PIPENV_BIN" ];
3647
then
@@ -46,7 +57,8 @@ request_install_to_virtual_env () {
4657
read ANSWER
4758
if [ "$ANSWER" == "yes" ];
4859
then
49-
pip install pipenv
60+
find_pip_bin
61+
$PIP_BIN install pipenv
5062
find_and_run_via_pip
5163
run "$PIPENV_BIN"
5264
else
@@ -59,16 +71,17 @@ request_install_to_user () {
5971
read ANSWER
6072
if [ "$ANSWER" == "yes" ];
6173
then
62-
pip install --user pipenv
74+
find_pip_bin
75+
$PIP_BIN install --user pipenv
6376
PIPENV_BIN="$(python3 -m site --user-base)/bin/pipenv"
6477
run "$PIPENV_BIN"
6578
else
6679
echo "Aborting"
6780
fi
6881
}
6982

70-
request_install () {
71-
IS_IN_VIRTUAL_ENV=$(python -c "import sys; print(hasattr(sys, 'real_prefix'))")
83+
request_install_and_run () {
84+
IS_IN_VIRTUAL_ENV=$(python3 -c "import sys; print(hasattr(sys, 'real_prefix'))")
7285
if [ "$IS_IN_VIRTUAL_ENV" == "True" ];
7386
then
7487
request_install_to_virtual_env
@@ -84,7 +97,8 @@ discover_pipenv_and_run() {
8497
then
8598
run "$PIPENV_BIN"
8699
else
87-
if pip show pipenv &> /dev/null;
100+
find_pip_bin
101+
if $PIP_BIN show pipenv &> /dev/null;
88102
then
89103
find_and_run_via_pip
90104
else
@@ -98,4 +112,4 @@ then
98112
run "$PIPENV_BIN"
99113
else
100114
discover_pipenv_and_run
101-
fi
115+
fi

‎exaslct_src/lib/data/image_info.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ def __init__(self,
3232

3333
class ImageInfo(Info):
3434
DOCKER_TAG_LENGTH_LIMIT=128
35-
MAX_TAG_SURPLUS = 20
35+
MAX_TAG_SURPLUS = 30
3636

3737
def __init__(self,
3838
source_repository_name: str, target_repository_name: str,

‎flavors/python3-ds-EXASOL-6.0.0/flavor_base/flavor_base_deps/packages/pip3_packages

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
pyexasol
22
keras
33
tensorflow
4+
tensorflow-hub
45
kmodes
56
seaborn
67
matplotlib
@@ -14,4 +15,5 @@ numpy
1415
scipy<=1.2.1 # due to https://github.com/CamDavidsonPilon/lifelines/issues/725
1516
scikit-learn
1617
pycurl
17-
statsmodels
18+
statsmodels
19+
PyYAML

‎flavors/python3-ds-EXASOL-6.1.0/flavor_base/flavor_base_deps/packages/pip3_packages

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
pyexasol
22
keras
33
tensorflow
4+
tensorflow-hub
45
kmodes
56
seaborn
67
matplotlib
@@ -14,4 +15,5 @@ numpy
1415
scipy<=1.2.1 # due to https://github.com/CamDavidsonPilon/lifelines/issues/725
1516
scikit-learn
1617
pycurl
17-
statsmodels
18+
statsmodels
19+
PyYAML

‎flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/base_test_build_run/Dockerfile

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
FROM {{language_deps}}
2+
3+
COPY --from={{base_test_deps}} /usr /usr
4+
COPY --from={{base_test_deps}} /lib /lib
5+
COPY --from={{base_test_deps}} /bin /bin
6+
COPY --from={{base_test_deps}} /opt /opt
7+
COPY --from={{base_test_deps}} /etc /etc
8+
COPY --from={{base_test_deps}} /env /env
9+
10+
RUN ldconfig
11+
12+
RUN mkdir /exaudf_src /exaudf
13+
COPY src/ /exaudf_src/
14+
15+
#ENV VERBOSE_BUILD "--subcommands --verbose_failures"
16+
17+
WORKDIR /exaudf_src/
18+
RUN ["/bin/bash", "-c", "source /env && bash build.sh -c dbg --define python=true //:exaudfclient_py3"]
19+
RUN cp -r -L bazel-bin/* /exaudf
20+
21+
RUN mkdir /exasol_emulator
22+
COPY emulator/ /exasol_emulator
23+
COPY src/exaudflib/zmqcontainer.proto /exasol_emulator
24+
RUN cd /exasol_emulator && protoc zmqcontainer.proto --python_out=.

‎flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/base_test_deps/Dockerfile

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
FROM {{build_deps}}
2+
3+
RUN mkdir -p /packages
4+
COPY base_test_deps/packages /packages/base_test_deps
5+
6+
RUN apt-get -y update && \
7+
apt-get -y install $(cat /packages/base_test_deps/apt_get_packages) && \
8+
locale-gen en_US.UTF-8 && \
9+
update-locale LC_ALL=en_US.UTF-8 && \
10+
apt-get -y clean && \
11+
apt-get -y autoremove && \
12+
ldconfig

‎flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/base_test_deps/packages/apt_get_packages

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
python-protobuf
2+
python-zmq
3+
gdb
4+
valgrind
5+
gdbserver
6+
binutils
7+
patchelf
8+
chrpath
9+
strace

‎flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/build_deps/Dockerfile

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
FROM {{cuda_deps}}
2+
ENV DEBIAN_FRONTEND=noninteractive
3+
4+
ENV ARCHIVE_UBUNTU_PREFIX=""
5+
RUN sed --in-place --regexp-extended "s/(\/\/)(archive\.ubuntu)/\1$ARCHIVE_UBUNTU_PREFIX\2/" /etc/apt/sources.list
6+
7+
COPY 01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
8+
RUN mkdir -p /packages
9+
COPY build_deps/packages /packages/build_deps
10+
11+
RUN apt-get -y update && \
12+
apt-get install -y $(cat /packages/build_deps/apt_get_packages) && \
13+
echo "deb [arch=amd64] http://storage.googleapis.com/bazel-apt stable jdk1.8" | tee /etc/apt/sources.list.d/bazel.list && \
14+
curl https://bazel.build/bazel-release.pub.gpg | apt-key add - && \
15+
apt-get update && \
16+
apt-get install -y bazel && \
17+
curl -L -o swig-2.0.4.tar.gz prdownloads.sourceforge.net/swig/swig-2.0.4.tar.gz && \
18+
tar zxf swig-2.0.4.tar.gz && \
19+
(cd swig-2.0.4 && ./configure --prefix=/usr && make && make install) && \
20+
rm -rf swig-2.0.4 swig-2.0.4.tar.gz && \
21+
locale-gen en_US.UTF-8 && \
22+
update-locale LC_ALL=en_US.UTF-8 && \
23+
apt-get -y clean && \
24+
apt-get -y autoremove && \
25+
ldconfig
26+
27+
RUN touch /env && \
28+
echo "export PROTOBUF_BIN=/usr/bin/protoc" >> /env && \
29+
echo "export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-amd64" >> /env

‎flavors/python3-ds-cuda-preview-EXASOL-6.1.0/flavor_base/build_deps/packages/apt_get_packages

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
software-properties-common
2+
coreutils
3+
locales
4+
tar
5+
curl
6+
openjdk-8-jdk
7+
build-essential
8+
locales
9+
libpcre3-dev
10+
protobuf-compiler

0 commit comments

Comments
 (0)