Skip to content

Commit c91e412

Browse files
fpetrini15fpetrini15
authored
test: Add L0_additional_dependency_dirs (#7707)
1 parent b88c3e9 commit c91e412

File tree

2 files changed

+284
-2
lines changed

2 files changed

+284
-2
lines changed

‎docs/customization_guide/deploy.md

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
<!--
2-
# Copyright (c) 2020-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# Copyright (c) 2020-2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
#
44
# Redistribution and use in source and binary forms, with or without
55
# modification, are permitted provided that the following conditions
@@ -244,7 +244,9 @@ to `False` if not required to avoid capturing or exposing any sensitive informat
244244

245245
#### `--trace-config level=<string> default "off"`
246246

247-
Tracing mode. Trace mode supports `triton` and `opentelemetry`. Unless required `--trace-config level=off` should be set to avoid capturing or exposing any sensitive information.
247+
Tracing mode. Trace mode supports `triton` and `opentelemetry`. Unless required
248+
`--trace-config level=off` should be set to avoid capturing or exposing any
249+
sensitive information.
248250

249251

250252
##### `backend-directory <string> default /opt/tritonserver/backends`
@@ -273,6 +275,17 @@ Directory where cache shared libraries are found.
273275
> must be access controlled. Adding untrusted files
274276
> can lead to arbitrarty code execution.
275277
278+
##### `backend-config=<backend>,additional-dependency-dirs=<string>`
279+
280+
This is an optional Windows feature that enables Triton to search custom
281+
dependency directories when loading a specific backend. The user can input
282+
these directories as a string of semicolon-separated paths (including a
283+
trailing semicolon). These directories are programmatically prepended to
284+
the process's PATH and are removed when the backend is loaded successfully.
285+
Windows will search PATH last in its search sequence, so be cautious that
286+
no untrusted files of same name exist in a location of higher search priority
287+
(e.g., System32). It is still recommended to add backend-specific dependencies
288+
to their corresponding backend folder when possible.
276289

277290

278291

‎qa/L0_additional_dependency_dirs/test.sh

Lines changed: 269 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,269 @@
1+
#!/bin/bash
2+
# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
#
4+
# Redistribution and use in source and binary forms, with or without
5+
# modification, are permitted provided that the following conditions
6+
# are met:
7+
# * Redistributions of source code must retain the above copyright
8+
# notice, this list of conditions and the following disclaimer.
9+
# * Redistributions in binary form must reproduce the above copyright
10+
# notice, this list of conditions and the following disclaimer in the
11+
# documentation and/or other materials provided with the distribution.
12+
# * Neither the name of NVIDIA CORPORATION nor the names of its
13+
# contributors may be used to endorse or promote products derived
14+
# from this software without specific prior written permission.
15+
#
16+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
17+
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18+
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
20+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
21+
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
22+
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
23+
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
24+
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27+
28+
REPO_VERSION=${NVIDIA_TRITON_SERVER_VERSION}
29+
if [ "$#" -ge 1 ]; then
30+
REPO_VERSION=$1
31+
fi
32+
if [ -z "$REPO_VERSION" ]; then
33+
echo -e "Repository version must be specified"
34+
echo -e "\n***\n*** Test Failed\n***"
35+
exit 1
36+
fi
37+
38+
if [[ -v WSL_DISTRO_NAME ]] || [[ -v MSYSTEM ]]; then
39+
TRITON_DIR=${TRITON_DIR:=c:/tritonserver}
40+
SERVER=${SERVER:=c:/tritonserver/bin/tritonserver.exe}
41+
BACKEND_DIR=${BACKEND_DIR:=c:/tritonserver/backends}
42+
MODELDIR=${MODELDIR:=c:/}
43+
TRITONSERVER_IPADDR=${TRITONSERVER_IPADDR:="localhost"}
44+
else
45+
echo -e "Test is not currently supported for Linux"
46+
exit 1
47+
fi
48+
49+
# FIXME: TPRD-244 - Do not use hard-coded dependency paths when TRT models
50+
# are being regularly generated on Windows.
51+
DEPENDENCY_PATH="C:/ci_test_deps/24.07"
52+
STALE_DEPENDENCY_PATH="C:/ci_test_deps/24.05"
53+
CUSTOM_DEPENDENCY_DIR=${MODELDIR}/custom_dependency_location
54+
STALE_DEPENDENCY_DIR=${MODELDIR}/stale_dependency_location
55+
TRT_MODEL_DIR="C:/tmp/24.07_trt_models"
56+
# Unlike the other commands, the mv command requires the path to be fully in
57+
# the UNIX style in order for regex to work properly. We cannot apply this
58+
# uniformly because the command line requires Windows path style.
59+
LOCAL_CI_TEST_DEPS_DIR=${MODELDIR_POSIX}/ci_test_deps
60+
61+
source ../common/util.sh
62+
rm -rf ${CUSTOM_DEPENDENCY_DIR} ${LOCAL_CI_TEST_DEPS_DIR} ${STALE_DEPENDENCY_DIR} ${MODELDIR_POSIX}/models
63+
rm -f ./*.log ./*.out
64+
RET=0;
65+
66+
mkdir ${LOCAL_CI_TEST_DEPS_DIR} ${CUSTOM_DEPENDENCY_DIR} ${STALE_DEPENDENCY_DIR}
67+
68+
# Make a copy of the ci_test_deps directory and move out the TRT dependencies
69+
# A pre-test step will add ${LOCAL_CI_TEST_DEPS_DIR} to the PATH
70+
cp -r ${DEPENDENCY_PATH}/* ${LOCAL_CI_TEST_DEPS_DIR} && mv ${LOCAL_CI_TEST_DEPS_DIR}/nvinfer* ${CUSTOM_DEPENDENCY_DIR}/
71+
72+
cp -r ${STALE_DEPENDENCY_PATH}/nvinfer* ${STALE_DEPENDENCY_DIR}/
73+
74+
mkdir ${MODELDIR_POSIX}/models && \
75+
cp -r ${TRT_MODEL_DIR}/qa_model_repository/plan_int32_int32_int32 ${MODELDIR_POSIX}/models/plan_int32_int32_int32
76+
77+
function simple_inference_check()
78+
{
79+
INFER_SUCCESS=1
80+
set +e
81+
code=`curl -s -w %{http_code} -o ./curl.out -d'{"inputs":[{"name":"INPUT0","datatype":"INT32","shape":[1,16],"data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]},{"name":"INPUT1","datatype":"INT32","shape":[1,16],"data":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16]}]}' ${TRITONSERVER_IPADDR}:8000/v2/models/plan_int32_int32_int32/infer`
82+
set -e
83+
if [ "$code" != "200" ]; then
84+
cat ./curl.out
85+
INFER_SUCCESS=0
86+
fi
87+
}
88+
89+
# Test Case 1: Run server when TRT implicit dependencies are not on path (expect FAIL)
90+
SERVER_LOG="./not_on_path_server.log"
91+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2"
92+
run_server
93+
if [ "$SERVER_PID" != "0" ]; then
94+
echo -e "\n***\n*** FAILED on line ${LINENO}: unexpected success starting $SERVER\n***"
95+
kill_server
96+
RET=1
97+
fi
98+
99+
# Test Case 2: Launch server with additional options to point to custom dependency location (expect SUCCESS)
100+
SERVER_LOG="./custom_dependency_dir_server.log"
101+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2 --backend-config=tensorrt,additional-dependency-dirs=${CUSTOM_DEPENDENCY_DIR};"
102+
run_server
103+
if [ "$SERVER_PID" == "0" ]; then
104+
echo -e "\n***\n*** Failed to start $SERVER\n***"
105+
cat $SERVER_LOG
106+
RET=1
107+
fi
108+
109+
simple_inference_check
110+
if [ "${INFER_SUCCESS}" == "0" ]; then
111+
echo -e "\n***\n*** FAILED on line ${LINENO}: simple inference check failed\n***"
112+
RET=1
113+
fi
114+
115+
kill_server
116+
117+
# Test Case 3: Launch server with additional options to point to custom dependency location and load model dynamically (expect SUCCESS)
118+
SERVER_LOG="./dynamic_loading_server.log"
119+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2 --backend-config=tensorrt,additional-dependency-dirs=${CUSTOM_DEPENDENCY_DIR}; --model-control-mode=explicit"
120+
run_server
121+
if [ "$SERVER_PID" == "0" ]; then
122+
echo -e "\n***\n*** Failed to start $SERVER\n***"
123+
cat $SERVER_LOG
124+
RET=1
125+
fi
126+
127+
code=`curl -s -w %{http_code} -o ./curl.out -X POST ${TRITONSERVER_IPADDR}:8000/v2/repository/models/plan_int32_int32_int32/load`
128+
if [ "$code" != "200" ]; then
129+
echo -e "\n***\n*** FAILED on line ${LINENO}: Unable to load model: plan_int32_int32_int32\n***"
130+
cat ./curl.out
131+
RET=1
132+
fi
133+
134+
simple_inference_check
135+
if [ "${INFER_SUCCESS}" == "0" ]; then
136+
echo -e "\n***\n*** FAILED on line ${LINENO}: simple inference check failed\n***"
137+
RET=1
138+
fi
139+
140+
kill_server
141+
142+
# Test Case 4: Run server when pointing to stale TRT dependencies (expect FAIL)
143+
SERVER_LOG="./stale_dependencies_server.log"
144+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2 --backend-config=tensorrt,additional-dependency-dirs=${STALE_DEPENDENCY_DIR};"
145+
run_server
146+
if [ "$SERVER_PID" != "0" ]; then
147+
echo -e "\n***\n*** FAILED on line ${LINENO}: unexpected success starting $SERVER\n***"
148+
kill_server
149+
RET=1
150+
fi
151+
152+
# Test Case 5: [Test ordering] Run server when pointing to stale and correct TRT dependencies (stale first - expect FAIL).
153+
SERVER_LOG="./stale_first_server.log"
154+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2 --backend-config=tensorrt,additional-dependency-dirs=${STALE_DEPENDENCY_DIR};${CUSTOM_DEPENDENCY_DIR};"
155+
run_server
156+
if [ "$SERVER_PID" != "0" ]; then
157+
echo -e "\n***\n*** FAILED on line ${LINENO}: unexpected success starting $SERVER\n***"
158+
kill_server
159+
RET=1
160+
fi
161+
162+
# Test Case 6: [Test ordering] Run server when pointing to stale and correct TRT dependencies (correct first - expect SUCCESS).
163+
SERVER_LOG="./correct_first_server.log"
164+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2 --backend-config=tensorrt,additional-dependency-dirs=${CUSTOM_DEPENDENCY_DIR};${STALE_DEPENDENCY_DIR};"
165+
run_server
166+
if [ "$SERVER_PID" == "0" ]; then
167+
echo -e "\n***\n*** Failed to start $SERVER\n***"
168+
cat $SERVER_LOG
169+
RET=1
170+
fi
171+
172+
simple_inference_check
173+
if [ "${INFER_SUCCESS}" == "0" ]; then
174+
echo -e "\n***\n*** FAILED on line ${LINENO}: simple inference check failed\n***"
175+
RET=1
176+
fi
177+
178+
kill_server
179+
180+
ORIGINAL_PATH=$PATH
181+
# Test Case 7: [Test ordering] Run server when correct TRT dependencies exist in environment PATH (expect SUCCESS)
182+
SERVER_LOG="./correct_in_environment.log"
183+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2"
184+
PATH="${ORIGINAL_PATH};${CUSTOM_DEPENDENCY_DIR};"
185+
run_server
186+
if [ "$SERVER_PID" == "0" ]; then
187+
echo -e "\n***\n*** Failed to start $SERVER\n***"
188+
cat $SERVER_LOG
189+
RET=1
190+
fi
191+
192+
simple_inference_check
193+
if [ "${INFER_SUCCESS}" == "0" ]; then
194+
echo -e "\n***\n*** FAILED on line ${LINENO}: simple inference check failed\n***"
195+
RET=1
196+
fi
197+
198+
PATH=$ORIGINAL_PATH
199+
kill_server
200+
201+
# Test Case 8: [Test ordering] Run server when correct TRT dependencies exist in environment PATH, but user specifies stale additional dependency directory (user input takes priority - expect FAIL)
202+
SERVER_LOG="./correct_in_environment_but_user_adds_stale.log"
203+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2 --backend-config=tensorrt,additional-dependency-dirs=${STALE_DEPENDENCY_DIR};"
204+
PATH="${ORIGINAL_PATH};${CUSTOM_DEPENDENCY_DIR};"
205+
run_server
206+
if [ "$SERVER_PID" != "0" ]; then
207+
echo -e "\n***\n*** FAILED on line ${LINENO}: unexpected success starting $SERVER\n***"
208+
kill_server
209+
RET=1
210+
fi
211+
212+
PATH=$ORIGINAL_PATH
213+
214+
# Test Case 9: [Test ordering] Run server when stale TRT dependencies exist in environment PATH (expect FAIL)
215+
SERVER_LOG="./stale_in_environment.log"
216+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2"
217+
PATH="${ORIGINAL_PATH};${STALE_DEPENDENCY_DIR};"
218+
run_server
219+
if [ "$SERVER_PID" != "0" ]; then
220+
echo -e "\n***\n*** FAILED on line ${LINENO}: unexpected success starting $SERVER\n***"
221+
kill_server
222+
RET=1
223+
fi
224+
225+
PATH=$ORIGINAL_PATH
226+
227+
# Test Case 10: [Test ordering] Run server when stale TRT dependencies exist in environment PATH, but user specifies correct additional dependency directory (user input takes priority - expect SUCCESS)
228+
SERVER_LOG="./stale_in_environment_but_user_adds_correct.log"
229+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2 --backend-config=tensorrt,additional-dependency-dirs=${CUSTOM_DEPENDENCY_DIR};"
230+
PATH="${ORIGINAL_PATH};${STALE_DEPENDENCY_DIR};"
231+
run_server
232+
if [ "$SERVER_PID" == "0" ]; then
233+
echo -e "\n***\n*** Failed to start $SERVER\n***"
234+
cat $SERVER_LOG
235+
RET=1
236+
fi
237+
238+
simple_inference_check
239+
if [ "${INFER_SUCCESS}" == "0" ]; then
240+
echo -e "\n***\n*** FAILED on line ${LINENO}: simple inference check failed\n***"
241+
RET=1
242+
fi
243+
244+
PATH=$ORIGINAL_PATH
245+
kill_server
246+
247+
# Test Case 11: Incorrect extension usage. User provided path(s) that are not semi-colon separated (expect FAIL).
248+
SERVER_LOG="./incorrect_usage_server.log"
249+
SERVER_ARGS="--model-repository=${MODELDIR}/models --backend-directory=${BACKEND_DIR} --log-verbose=2 --backend-config=tensorrt,additional-dependency-dirs=C:/not_semicolon_terminated"
250+
run_server
251+
if [ "$SERVER_PID" != "0" ]; then
252+
echo -e "\n***\n*** FAILED on line ${LINENO}: unexpected success starting $SERVER\n***"
253+
kill_server
254+
RET=1
255+
fi
256+
257+
if [ $(cat ${SERVER_LOG} | grep "malformed" | wc -l) -eq 0 ]; then
258+
echo -e "\n***\n*** FAILED on line ${LINENO}: expected error statement not found $SERVER\n***"
259+
cat $SERVER_LOG
260+
RET=1
261+
fi
262+
263+
if [ $RET -eq 0 ]; then
264+
echo -e "\n***\n*** Test Passed\n***"
265+
else
266+
echo -e "\n***\n*** Test FAILED\n***"
267+
fi
268+
269+
exit $RET

0 commit comments

Comments
 (0)