Skip to content

Commit ec8344a

Browse files
GuanLuomc-nv
authored andcommitted
Python binding of Triton server C API (#265)
* Add Python binding for Triton server API * Use uintptr_t for pointer. Address exception name. Interact with GIL * Fix lifetime dependency between request and server * Better variable name. Pass by reference to avoid extra copy construct * Add wheel build script. Address comment * Format * Avoid core referring to PyBind in backend build * Using generator expression to build on Windows * Specify rpath for runtime library lookup. Fix bug * Address comment. Format
1 parent d300a9a commit ec8344a

File tree

10 files changed

+3649
-0
lines changed

10 files changed

+3649
-0
lines changed

CMakeLists.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -415,3 +415,7 @@ export(
415415
)
416416

417417
export(PACKAGE TritonCore)
418+
419+
if(NOT TRITON_CORE_HEADERS_ONLY)
420+
add_subdirectory(python python)
421+
endif()

python/CMakeLists.txt

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# Redistribution and use in source and binary forms, with or without
4+
# modification, are permitted provided that the following conditions
5+
# are met:
6+
# * Redistributions of source code must retain the above copyright
7+
# notice, this list of conditions and the following disclaimer.
8+
# * Redistributions in binary form must reproduce the above copyright
9+
# notice, this list of conditions and the following disclaimer in the
10+
# documentation and/or other materials provided with the distribution.
11+
# * Neither the name of NVIDIA CORPORATION nor the names of its
12+
# contributors may be used to endorse or promote products derived
13+
# from this software without specific prior written permission.
14+
#
15+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
16+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
19+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26+
cmake_minimum_required(VERSION 3.18)
27+
28+
add_subdirectory(tritonserver)
29+
30+
file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION ${TRITON_VERSION})
31+
configure_file(../LICENSE LICENSE.txt COPYONLY)
32+
configure_file(setup.py setup.py @ONLY)
33+
file(COPY test/test_binding.py DESTINATION ./test/.)
34+
35+
set(WHEEL_DEPENDS
36+
${CMAKE_CURRENT_BINARY_DIR}/TRITON_VERSION
37+
${CMAKE_CURRENT_BINARY_DIR}/LICENSE.txt
38+
${CMAKE_CURRENT_BINARY_DIR}/setup.py
39+
${CMAKE_CURRENT_BINARY_DIR}/tritonserver
40+
python-bindings
41+
)
42+
43+
set(wheel_stamp_file "stamp.whl")
44+
45+
add_custom_command(
46+
OUTPUT "${wheel_stamp_file}"
47+
COMMAND python3
48+
ARGS
49+
"${CMAKE_CURRENT_SOURCE_DIR}/build_wheel.py"
50+
--dest-dir "${CMAKE_CURRENT_BINARY_DIR}/generic"
51+
--binding-path $<TARGET_FILE:python-bindings>
52+
DEPENDS ${WHEEL_DEPENDS}
53+
)
54+
55+
add_custom_target(
56+
generic-server-wheel ALL
57+
DEPENDS
58+
"${wheel_stamp_file}"
59+
)
60+
61+
install(
62+
CODE "file(GLOB _Wheel \"${CMAKE_CURRENT_BINARY_DIR}/generic/triton*.whl\")"
63+
CODE "file(INSTALL \${_Wheel} DESTINATION \"${CMAKE_INSTALL_PREFIX}/python\")"
64+
)
65+
66+
# Test
67+
install(
68+
CODE "file(INSTALL ${CMAKE_CURRENT_BINARY_DIR}/test/test_binding.py DESTINATION \"${CMAKE_INSTALL_PREFIX}/python\")"
69+
)

python/build_wheel.py

Lines changed: 124 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+
import argparse
29+
import os
30+
import pathlib
31+
import re
32+
import shutil
33+
import subprocess
34+
import sys
35+
from distutils.dir_util import copy_tree
36+
from tempfile import mkstemp
37+
38+
39+
def fail_if(p, msg):
40+
if p:
41+
print("error: {}".format(msg), file=sys.stderr)
42+
sys.exit(1)
43+
44+
45+
def mkdir(path):
46+
pathlib.Path(path).mkdir(parents=True, exist_ok=True)
47+
48+
49+
def touch(path):
50+
pathlib.Path(path).touch()
51+
52+
53+
def cpdir(src, dest):
54+
copy_tree(src, dest, preserve_symlinks=1)
55+
56+
57+
def sed(pattern, replace, source, dest=None):
58+
fin = open(source, "r")
59+
if dest:
60+
fout = open(dest, "w")
61+
else:
62+
fd, name = mkstemp()
63+
fout = open(name, "w")
64+
65+
for line in fin:
66+
out = re.sub(pattern, replace, line)
67+
fout.write(out)
68+
69+
fin.close()
70+
fout.close()
71+
if not dest:
72+
shutil.copyfile(name, source)
73+
74+
75+
if __name__ == "__main__":
76+
parser = argparse.ArgumentParser()
77+
78+
parser.add_argument(
79+
"--dest-dir", type=str, required=True, help="Destination directory."
80+
)
81+
parser.add_argument(
82+
"--binding-path", type=str, required=True, help="Path to Triton Python binding."
83+
)
84+
85+
FLAGS = parser.parse_args()
86+
87+
FLAGS.triton_version = None
88+
with open("TRITON_VERSION", "r") as vfile:
89+
FLAGS.triton_version = vfile.readline().strip()
90+
91+
FLAGS.whl_dir = os.path.join(FLAGS.dest_dir, "wheel")
92+
93+
print("=== Building in: {}".format(os.getcwd()))
94+
print("=== Using builddir: {}".format(FLAGS.whl_dir))
95+
print("Adding package files")
96+
97+
mkdir(os.path.join(FLAGS.whl_dir, "tritonserver"))
98+
shutil.copy("tritonserver/__init__.py", os.path.join(FLAGS.whl_dir, "tritonserver"))
99+
100+
cpdir("tritonserver/_c", os.path.join(FLAGS.whl_dir, "tritonserver", "_c"))
101+
PYBIND_LIB = os.path.basename(FLAGS.binding_path)
102+
shutil.copyfile(
103+
FLAGS.binding_path,
104+
os.path.join(FLAGS.whl_dir, "tritonserver", "_c", PYBIND_LIB),
105+
)
106+
107+
shutil.copyfile("LICENSE.txt", os.path.join(FLAGS.whl_dir, "LICENSE.txt"))
108+
shutil.copyfile("setup.py", os.path.join(FLAGS.whl_dir, "setup.py"))
109+
110+
os.chdir(FLAGS.whl_dir)
111+
print("=== Building wheel")
112+
args = ["python3", "setup.py", "bdist_wheel"]
113+
114+
wenv = os.environ.copy()
115+
wenv["VERSION"] = FLAGS.triton_version
116+
wenv["TRITON_PYBIND"] = PYBIND_LIB
117+
p = subprocess.Popen(args, env=wenv)
118+
p.wait()
119+
fail_if(p.returncode != 0, "setup.py failed")
120+
121+
cpdir("dist", FLAGS.dest_dir)
122+
123+
print("=== Output wheel file is in: {}".format(FLAGS.dest_dir))
124+
touch(os.path.join(FLAGS.dest_dir, "stamp.whl"))

python/setup.py

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/usr/bin/env python3
2+
# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+
import os
29+
import sys
30+
from itertools import chain
31+
32+
from setuptools import find_packages, setup
33+
34+
if "--plat-name" in sys.argv:
35+
PLATFORM_FLAG = sys.argv[sys.argv.index("--plat-name") + 1]
36+
else:
37+
PLATFORM_FLAG = "any"
38+
39+
if "VERSION" not in os.environ:
40+
raise Exception("envvar VERSION must be specified")
41+
42+
VERSION = os.environ["VERSION"]
43+
44+
try:
45+
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
46+
47+
class bdist_wheel(_bdist_wheel):
48+
def finalize_options(self):
49+
_bdist_wheel.finalize_options(self)
50+
self.root_is_pure = False
51+
52+
def get_tag(self):
53+
pyver, abi, plat = "py3", "none", PLATFORM_FLAG
54+
return pyver, abi, plat
55+
56+
except ImportError:
57+
bdist_wheel = None
58+
59+
this_directory = os.path.abspath(os.path.dirname(__file__))
60+
61+
data_files = [
62+
("", ["LICENSE.txt"]),
63+
]
64+
platform_package_data = [os.environ["TRITON_PYBIND"]]
65+
66+
setup(
67+
name="tritonserver",
68+
version=VERSION,
69+
author="NVIDIA Inc.",
70+
author_email="sw-dl-triton@nvidia.com",
71+
description="Python API of the Triton In-Process Server",
72+
license="BSD",
73+
url="https://developer.nvidia.com/nvidia-triton-inference-server",
74+
classifiers=[
75+
"Development Status :: 5 - Production/Stable",
76+
"Intended Audience :: Developers",
77+
"Intended Audience :: Science/Research",
78+
"Intended Audience :: Information Technology",
79+
"Topic :: Scientific/Engineering",
80+
"Topic :: Scientific/Engineering :: Image Recognition",
81+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
82+
"Topic :: Software Development :: Libraries",
83+
"Topic :: Utilities",
84+
"License :: OSI Approved :: BSD License",
85+
"Programming Language :: Python :: 3",
86+
"Programming Language :: Python :: 3.10",
87+
"Environment :: Console",
88+
"Natural Language :: English",
89+
"Operating System :: OS Independent",
90+
],
91+
packages=find_packages(),
92+
package_data={
93+
"": platform_package_data,
94+
},
95+
zip_safe=False,
96+
cmdclass={"bdist_wheel": bdist_wheel},
97+
data_files=data_files,
98+
)

0 commit comments

Comments
 (0)