Skip to content

Commit b47e95c

Browse files
mcollins42mcollins42
authored
Add tensorflow-gcs-config to the tensorflow-whl build (#784)
Add a copy of tensorflow-gcs-config source and modify tensorflow-whl/Dockerfile to build the tensorflow_gcs_config wheel. http://b/152051681
1 parent 28d87fe commit b47e95c

File tree

18 files changed

+962
-1
lines changed

18 files changed

+962
-1
lines changed

‎tensorflow-whl/CHANGELOG.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,5 @@
1212
* `2.1.0-py36`: TensorFlow 2.1.0 with Python 3.6
1313
* `2.1.0-py36-2`: TensorFlow 2.1.0 with CUDA 10.1
1414
* `2.1.0-py37`: TensorFlow 2.1.0 with Python 3.7
15-
* `2.1.0-py37-2`: TensorFlow 2.1.0 with Python 3.7 & DLVM base image.
15+
* `2.1.0-py37-2`: TensorFlow 2.1.0 with Python 3.7 & DLVM base image.
16+
* `2.1.0-py37-3`: TensorFlow 2.1.0 with Python 3.7, DLVM base image, tensorflow-gcs-config.

‎tensorflow-whl/Dockerfile

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,5 +100,14 @@ RUN cd /usr/local/src/tensorflow && \
100100
bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/tensorflow_gpu && \
101101
bazel clean
102102

103+
ADD tensorflow-gcs-config /usr/local/src/tensorflow_gcs_config/
104+
105+
# Build tensorflow_gcs_config library against the tensorflow_cpu build
106+
RUN cd /usr/local/src/tensorflow_gcs_config && \
107+
apt-get install -y libcurl4-openssl-dev && \
108+
pip install /tmp/tensorflow_cpu/tensorflow*.whl && \
109+
python setup.py bdist_wheel -d /tmp/tensorflow_gcs_config && \
110+
bazel clean
111+
103112
# Print out the built .whl files
104113
RUN ls -R /tmp/tensorflow*

‎tensorflow-whl/tensorflow-gcs-config/.gitignore

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
bazel-bin
2+
bazel-genfiles
3+
bazel-out
4+
bazel-tensorflow-gcs-config
5+
bazel-out
6+
bazel-testlogs
7+
build
8+
dist
9+
__pycache__
10+
*.egg-info
11+
*.so
12+
.bazelrc

‎tensorflow-whl/tensorflow-gcs-config/Dockerfile

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
FROM tensorflow/tensorflow:custom-op
2+
3+
ARG TF_VERSION
4+
ARG UID
5+
ARG GID
6+
ARG USERNAME="build"
7+
ARG CONDA_ADD_PACKAGES=""
8+
ARG BAZEL_VERSION=0.24.1
9+
ARG BAZEL_OS=linux
10+
11+
RUN apt-get update && \
12+
apt-get install -y \
13+
git \
14+
curl \
15+
nano \
16+
unzip \
17+
ffmpeg \
18+
dnsutils
19+
20+
RUN groupadd -g ${GID} ${USERNAME}
21+
RUN useradd -d /home/${USERNAME} -ms /bin/bash -g ${USERNAME} -G root -u $UID ${USERNAME}
22+
USER ${USERNAME}
23+
24+
WORKDIR /home/${USERNAME}
25+
26+
RUN curl -sL https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-${BAZEL_OS}-x86_64.sh -o bazel-install.sh && \
27+
bash -x bazel-install.sh --user && \
28+
rm bazel-install.sh
29+
30+
ARG CONDA_OS=Linux
31+
32+
# Miniconda - Python 3.6, 64-bit, x86, latest
33+
RUN curl -sL https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh -o mconda-install.sh && \
34+
bash -x mconda-install.sh -b -p miniconda && \
35+
rm mconda-install.sh
36+
37+
ENV PATH="/home/${USERNAME}/miniconda/bin:$PATH"
38+
39+
RUN conda create -y -q -n tensorflow-gcs-config python=3.6 ${CONDA_ADD_PACKAGES}
40+
41+
RUN echo ". /miniconda/etc/profile.d/conda.sh" >> ~/.bashrc && \
42+
echo "source activate tensorflow-gcs-config" >> ~/.bashrc
43+
44+
ARG PIP_ADD_PACKAGES=""
45+
46+
RUN /bin/bash -c "source activate tensorflow-gcs-config && python -m pip install -U \
47+
pytest \
48+
pylint \
49+
boto3 \
50+
twine \
51+
google-cloud-pubsub==0.39.1 \
52+
pandas \
53+
fastavro \
54+
'tensorflow>=2' \
55+
${PIP_ADD_PACKAGES} \
56+
"
57+
58+
# This just forces a new fetch of the latest TF binary if the version changes.
59+
RUN /bin/bash -c "echo ${TF_VERSION}"
60+
RUN /bin/bash -c "source activate tensorflow-gcs-config && python -m pip install -U 'tensorflow>=2'"
61+
62+
RUN bazel help > /dev/null

‎tensorflow-whl/tensorflow-gcs-config/MANIFEST.in

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
include tensorflow_gcs_config/*.py
2+
include tensorflow_gcs_config/*.so

‎tensorflow-whl/tensorflow-gcs-config/README.md

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
# Tensorflow GCS Configuration Ops
2+
3+
This package provides TF 2.X compatible versions of the
4+
`tf.contrib.cloud.configure_gcs()` operations.
5+
6+
This is a slightly modified and repackaged version of the GCS code in TensorFlow I/O,
7+
in particular the [tfio.gcs](https://www.tensorflow.org/io/api_docs/python/tfio/gcs) module.
8+
9+
This is a copy of the internal source released as the
10+
[tensorflow-gcs-config](https://pypi.org/project/tensorflow-gcs-config/) package.

‎tensorflow-whl/tensorflow-gcs-config/WORKSPACE

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
workspace(name = "tensorflow_gcs_config")
2+
3+
load("//third_party/tensorflow:tf_configure.bzl", "tf_configure")
4+
load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
5+
6+
tf_configure(
7+
name = "local_config_tf",
8+
)
9+
10+
http_archive(
11+
name = "com_github_grpc_grpc",
12+
sha256 = "1bf082fb3016154d3f806da8eb5876caf05743da4b2e8130fadd000df74b5bb6",
13+
strip_prefix = "grpc-1.21.1",
14+
urls = [
15+
"https://mirror.bazel.build/github.com/grpc/grpc/archive/v1.21.1.tar.gz",
16+
"https://github.com/grpc/grpc/archive/v1.21.1.tar.gz",
17+
],
18+
)
19+
20+
# 3.7.1 with a fix to BUILD file
21+
http_archive(
22+
name = "com_google_protobuf",
23+
sha256 = "1c020fafc84acd235ec81c6aac22d73f23e85a700871466052ff231d69c1b17a",
24+
strip_prefix = "protobuf-5902e759108d14ee8e6b0b07653dac2f4e70ac73",
25+
urls = [
26+
"http://mirror.tensorflow.org/github.com/protocolbuffers/protobuf/archive/5902e759108d14ee8e6b0b07653dac2f4e70ac73.tar.gz",
27+
"https://github.com/protocolbuffers/protobuf/archive/5902e759108d14ee8e6b0b07653dac2f4e70ac73.tar.gz",
28+
],
29+
)
30+
31+
load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps")
32+
33+
grpc_deps()
34+
35+
http_archive(
36+
name = "jsoncpp_git",
37+
build_file = "//third_party:jsoncpp.BUILD",
38+
sha256 = "c49deac9e0933bcb7044f08516861a2d560988540b23de2ac1ad443b219afdb6",
39+
strip_prefix = "jsoncpp-1.8.4",
40+
urls = [
41+
"http://mirror.tensorflow.org/github.com/open-source-parsers/jsoncpp/archive/1.8.4.tar.gz",
42+
"https://github.com/open-source-parsers/jsoncpp/archive/1.8.4.tar.gz",
43+
],
44+
)

‎tensorflow-whl/tensorflow-gcs-config/build.py

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
# =============================================================================
15+
16+
from __future__ import print_function
17+
18+
import os
19+
import re
20+
import sys
21+
import tensorflow as tf
22+
23+
def write_config():
24+
"""Retrive compile and link information from tensorflow and write to .bazelrc."""
25+
26+
cflags = tf.sysconfig.get_compile_flags()
27+
28+
inc_regex = re.compile("^-I")
29+
opt_regex = re.compile("^-D")
30+
31+
include_list = []
32+
opt_list = []
33+
34+
for arg in cflags:
35+
if inc_regex.match(arg):
36+
include_list.append(arg)
37+
elif opt_regex.match(arg):
38+
opt_list.append(arg)
39+
else:
40+
print("WARNING: Unexpected cflag item {}".format(arg))
41+
42+
43+
if len(include_list) != 1:
44+
print("ERROR: Expected a single include directory in " +
45+
"tf.sysconfig.get_compile_flags()")
46+
exit(1)
47+
48+
49+
library_regex = re.compile("^-l")
50+
libdir_regex = re.compile("^-L")
51+
52+
library_list = []
53+
libdir_list = []
54+
55+
lib = tf.sysconfig.get_link_flags()
56+
57+
for arg in lib:
58+
if library_regex.match(arg):
59+
library_list.append(arg)
60+
elif libdir_regex.match(arg):
61+
libdir_list.append(arg)
62+
else:
63+
print("WARNING: Unexpected link flag item {}".format(arg))
64+
65+
if len(library_list) != 1 or len(libdir_list) != 1:
66+
print("ERROR: Expected exactly one lib and one libdir in" +
67+
"tf.sysconfig.get_link_flags()")
68+
exit(1)
69+
70+
try:
71+
72+
with open(".bazelrc", "w") as bazel_rc:
73+
for opt in opt_list:
74+
bazel_rc.write('build --copt="{}"\n'.format(opt))
75+
76+
bazel_rc.write('build --action_env TF_HEADER_DIR="{}"\n'
77+
.format(include_list[0][2:]))
78+
79+
bazel_rc.write('build --action_env TF_SHARED_LIBRARY_DIR="{}"\n'
80+
.format(libdir_list[0][2:]))
81+
library_name = library_list[0][2:]
82+
if library_name.startswith(":"):
83+
library_name = library_name[1:]
84+
else:
85+
library_name = "lib" + library_name + ".so"
86+
bazel_rc.write('build --action_env TF_SHARED_LIBRARY_NAME="{}"\n'
87+
.format(library_name))
88+
bazel_rc.close()
89+
except OSError:
90+
print("ERROR: Writing .bazelrc")
91+
exit(1)
92+
93+
94+
def compile_bazel():
95+
write_config()
96+
97+
if os.system('rm -f tensorflow_gcs_config/*.so && bazel build -c dbg //tensorflow_gcs_config:_gcs_config_ops.so && cp bazel-bin/tensorflow_gcs_config/_gcs_config_ops.so tensorflow_gcs_config/') != 0:
98+
raise Exception('Failed to build C extension.')

‎tensorflow-whl/tensorflow-gcs-config/setup.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
from setuptools import setup
2+
3+
4+
setup_kwargs = {
5+
}
6+
7+
from build import compile_bazel
8+
compile_bazel()
9+
10+
setup(
11+
name='tensorflow-gcs-config',
12+
version='2.1.7',
13+
description='TensorFlow operations for configuring access to GCS (Google Compute Storage) resources.',
14+
long_description='TensorFlow operations for configuring access to GCS (Google Compute Storage) resources.',
15+
author='Google, Inc.',
16+
author_email=None,
17+
url=None,
18+
packages = ['tensorflow_gcs_config'],
19+
include_package_data=True,
20+
)

‎tensorflow-whl/tensorflow-gcs-config/tensorflow_gcs_config/BUILD

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
licenses(["notice"]) # Apache 2.0
2+
3+
package(default_visibility = ["//visibility:public"])
4+
5+
cc_binary(
6+
name = "_gcs_config_ops.so",
7+
srcs = [
8+
"gcs_config_op_kernels.cc",
9+
"gcs_config_ops.cc",
10+
],
11+
copts = [
12+
"-pthread",
13+
"-std=c++11",
14+
"-DNDEBUG",
15+
],
16+
linkshared = 1,
17+
deps = [
18+
"@jsoncpp_git//:jsoncpp",
19+
"@local_config_tf//:libtensorflow_framework",
20+
"@local_config_tf//:tf_header_lib",
21+
],
22+
)

0 commit comments

Comments
 (0)