Update main branch post 24.11 (#7829)

Co-authored-by: Anant Sharma <anants@nvidia.com>
Co-authored-by: Jacky <18255193+kthui@users.noreply.github.com>
Co-authored-by: Kris Hung <krish@nvidia.com>

Author: 4 people

CMakeLists.txt

@@ -221,7 +221,7 @@ ExternalProject_Add(triton-server
     -Dre2_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/re2/${LIB_DIR}/cmake/re2
     -Dabsl_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/absl/${LIB_DIR}/cmake/absl
     -DCURL_DIR:STRING=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/curl/${LIB_DIR}/cmake/CURL
-    -Dnlohmann_json_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/nlohmann_json/${LIB_DIR}/cmake/nlohmann_json
+    -Dnlohmann_json_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/nlohmann_json/share/cmake/nlohmann_json
     -DLibevent_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevent/lib/cmake/libevent
     -Dlibevhtp_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/libevhtp/lib/cmake/libevhtp
     -Dstorage_client_DIR:PATH=${TRITON_THIRD_PARTY_INSTALL_PREFIX}/google-cloud-cpp/${LIB_DIR}/cmake/storage_client

Dockerfile.QA

@@ -59,20 +59,19 @@ RUN apt-get update && \
             libboost-dev \
             python3-dev \
             python3-pip \
+            python3-wheel \
+            python3-setuptools \
             rapidjson-dev \
             software-properties-common && \
     rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install --upgrade pip && \
-    pip3 install --upgrade wheel setuptools
-
 RUN apt update -q=2 \
     && apt install -y gpg wget \
     && wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - |  tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null \
     && . /etc/os-release \
     && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get update -q=2 \
-    && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7*
+    && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3*
 
 # Add inception_graphdef model to example repo
 WORKDIR /workspace/docs/examples/model_repository
@@ -311,20 +310,12 @@ ARG TARGETPLATFORM
 ENV DEBIAN_FRONTEND=noninteractive
 
 # install platform specific packages
-RUN if [ $(cat /etc/os-release | grep 'VERSION_ID="20.04"' | wc -l) -ne 0 ]; then \
-        apt-get update && \
-        apt-get install -y --no-install-recommends \
-                libpng-dev; \
-    elif [ $(cat /etc/os-release | grep 'VERSION_ID="22.04"' | wc -l) -ne 0 ]; then \
-        apt-get update && \
-        apt-get install -y --no-install-recommends \
-                libpng-dev; \
-    elif [ $(cat /etc/os-release | grep 'VERSION_ID="18.04"' | wc -l) -ne 0 ]; then \
+RUN if grep -qE '^VERSION_ID="(18\.04|20\.04|22\.04|24\.04)' /etc/os-release; then \
         apt-get update && \
         apt-get install -y --no-install-recommends \
                 libpng-dev; \
     else \
-        echo "Ubuntu version must be either 18.04, 20.04 or 22.04" && \
+        echo "Ubuntu version must be either 18.04, 20.04, 22.04 or 24.04" && \
         exit 1; \
     fi
 
@@ -337,14 +328,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
                               libarchive-dev \
                               libopencv-core-dev \
                               libzmq3-dev \
-                              maven \
                               openjdk-11-jdk \
                               nginx \
                               npm \
                               protobuf-compiler \
                               python3-dev \
                               python3-pip \
                               python3-protobuf \
+                              python3-wheel \
                               python3-setuptools \
                               swig \
                               valgrind && \
@@ -354,9 +345,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 RUN rm -f /usr/bin/python && \
     ln -s /usr/bin/python3 /usr/bin/python
 
-RUN pip3 install --upgrade wheel setuptools && \
-    pip3 install --upgrade "numpy<2" pillow attrdict future grpcio requests gsutil \
-                           awscli six grpcio-channelz prettytable virtualenv \
+RUN pip3 install --upgrade "numpy<2" pillow attrdict future "grpcio<1.68" requests gsutil \
+                           awscli six "grpcio-channelz<1.68" prettytable virtualenv \
                            check-jsonschema
 
 # go needed for example go client test.

Dockerfile.sdk

@@ -29,7 +29,7 @@
 #
 
 # Base image on the minimum Triton container
-ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:24.10-py3-min
+ARG BASE_IMAGE=nvcr.io/nvidia/tritonserver:24.11-py3-min
 
 ARG TRITON_CLIENT_REPO_SUBDIR=clientrepo
 ARG TRITON_PA_REPO_SUBDIR=perfanalyzerrepo
@@ -44,7 +44,7 @@ ARG JAVA_BINDINGS_MAVEN_VERSION=3.8.4
 ARG JAVA_BINDINGS_JAVACPP_PRESETS_TAG=1.5.8
 
 # DCGM version to install for Model Analyzer
-ARG DCGM_VERSION=3.2.6
+ARG DCGM_VERSION=3.3.6
 
 ARG NVIDIA_TRITON_SERVER_SDK_VERSION=unknown
 ARG NVIDIA_BUILD_ID=unknown
@@ -57,6 +57,7 @@ FROM ${BASE_IMAGE} AS sdk_build
 
 # Ensure apt-get won't prompt for selecting options
 ENV DEBIAN_FRONTEND=noninteractive
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
 
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
@@ -78,15 +79,15 @@ RUN apt-get update && \
             python3 \
             python3-pip \
             python3-dev \
+            python3-wheel \
+            python3-setuptools \
             rapidjson-dev \
             vim \
             wget \
             python3-pdfkit \
             openjdk-11-jdk \
             maven && \
-    pip3 install --upgrade wheel setuptools && \
-    pip3 install --upgrade grpcio-tools && \
-    pip3 install --upgrade pip
+    pip3 install --upgrade "grpcio-tools<1.68"
 
 # Client build requires recent version of CMake (FetchContent required)
 # Using CMAKE installation instruction from:: https://apt.kitware.com/
@@ -96,7 +97,7 @@ RUN apt update -q=2 \
     && . /etc/os-release \
     && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \
     && apt-get update -q=2 \
-    && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7* \
+    && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3* \
     && cmake --version
 
 # Build expects "python" executable (not python3).
@@ -183,6 +184,7 @@ FROM ${BASE_IMAGE}
 
 # Ensure apt-get won't prompt for selecting options
 ENV DEBIAN_FRONTEND=noninteractive
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
 
 ARG DCGM_VERSION
 ARG TRITON_REPO_ORGANIZATION
@@ -205,14 +207,14 @@ RUN apt-get update && \
             python3 \
             python3-pip \
             python3-dev \
+            python3-wheel \
+            python3-setuptools \
             vim \
             wget \
             python3-pdfkit \
             maven \
             default-jdk && \
-    pip3 install --upgrade wheel setuptools && \
-    pip3 install --upgrade grpcio-tools && \
-    pip3 install --upgrade pip
+    pip3 install "grpcio<1.68" "grpcio-tools<1.68"
 
 WORKDIR /workspace
 COPY TRITON_VERSION .
@@ -250,7 +252,7 @@ RUN pip3 install install/python/genai_perf-*.whl
 RUN if [ "$TRITON_ENABLE_GPU" = "ON" ]; then \
         [ "$(uname -m)" != "x86_64" ] && arch="sbsa" || arch="x86_64" && \
         curl -o /tmp/cuda-keyring.deb \
-        https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/$arch/cuda-keyring_1.0-1_all.deb \
+        https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/$arch/cuda-keyring_1.1-1_all.deb \
         && apt install /tmp/cuda-keyring.deb && rm /tmp/cuda-keyring.deb && \
         apt-get update && apt-get install -y datacenter-gpu-manager=1:${DCGM_VERSION}; \
     fi

README.md

@@ -32,8 +32,8 @@
 
 >[!WARNING]
 >You are currently on the `main` branch which tracks under-development progress
->towards the next release. The current release is version [2.51.0](https://github.com/triton-inference-server/server/releases/latest)
->and corresponds to the 24.10 container release on NVIDIA GPU Cloud (NGC).
+>towards the next release. The current release is version [2.52.0](https://github.com/triton-inference-server/server/releases/latest)
+>and corresponds to the 24.11 container release on NVIDIA GPU Cloud (NGC).
 
 Triton Inference Server is an open source inference serving software that
 streamlines AI inferencing. Triton enables teams to deploy any AI model from
@@ -91,16 +91,16 @@ Inference Server with the
 
 ```bash
 # Step 1: Create the example model repository
-git clone -b r24.10 https://github.com/triton-inference-server/server.git
+git clone -b r24.11 https://github.com/triton-inference-server/server.git
 cd server/docs/examples
 ./fetch_models.sh
 
 # Step 2: Launch triton from the NGC Triton container
-docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:24.10-py3 tritonserver --model-repository=/models
+docker run --gpus=1 --rm --net=host -v ${PWD}/model_repository:/models nvcr.io/nvidia/tritonserver:24.11-py3 tritonserver --model-repository=/models
 
 # Step 3: Sending an Inference Request
 # In a separate console, launch the image_client example from the NGC Triton SDK container
-docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:24.10-py3-sdk
+docker run -it --rm --net=host nvcr.io/nvidia/tritonserver:24.11-py3-sdk
 /workspace/install/bin/image_client -m densenet_onnx -c 3 -s INCEPTION /workspace/images/mug.jpg
 
 # Inference should return the following

TRITON_VERSION

@@ -1 +1 @@
-2.53.0dev
+2.53.0dev

build.py

@@ -73,11 +73,11 @@
 DEFAULT_TRITON_VERSION_MAP = {
     "release_version": "2.53.0dev",
     "triton_container_version": "24.12dev",
-    "upstream_container_version": "24.10",
+    "upstream_container_version": "24.11",
     "ort_version": "1.19.2",
     "ort_openvino_version": "2024.4.0",
     "standalone_openvino_version": "2024.4.0",
-    "dcgm_version": "3.2.6",
+    "dcgm_version": "3.3.6",
     "vllm_version": "0.5.5",
     "rhel_py_version": "3.12.3",
 }
@@ -885,7 +885,7 @@ def install_dcgm_libraries(dcgm_version, target_machine):
 ENV DCGM_VERSION {}
 # Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads
 RUN curl -o /tmp/cuda-keyring.deb \\
-        https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/sbsa/cuda-keyring_1.0-1_all.deb \\
+        https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/sbsa/cuda-keyring_1.1-1_all.deb \\
       && apt install /tmp/cuda-keyring.deb \\
       && rm /tmp/cuda-keyring.deb \\
       && apt-get update \\
@@ -898,7 +898,7 @@ def install_dcgm_libraries(dcgm_version, target_machine):
 ENV DCGM_VERSION {}
 # Install DCGM. Steps from https://developer.nvidia.com/dcgm#Downloads
 RUN curl -o /tmp/cuda-keyring.deb \\
-          https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2204/x86_64/cuda-keyring_1.0-1_all.deb \\
+          https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \\
       && apt install /tmp/cuda-keyring.deb \\
       && rm /tmp/cuda-keyring.deb \\
       && apt-get update \\
@@ -1035,6 +1035,7 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap):
         df += """
 # Ensure apt-get won't prompt for selecting options
 ENV DEBIAN_FRONTEND=noninteractive
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
 
 # Install docker docker buildx
 RUN apt-get update \\
@@ -1071,6 +1072,7 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap):
             patchelf \\
             python3-dev \\
             python3-pip \\
+            python3-wheel \\
             python3-setuptools \\
             rapidjson-dev \\
             scons \\
@@ -1085,10 +1087,7 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap):
             wget \\
       && rm -rf /var/lib/apt/lists/*
 
-RUN pip3 install --upgrade pip \\
-      && pip3 install --upgrade \\
-          wheel \\
-          setuptools \\
+RUN pip3 install --upgrade \\
           docker \\
           virtualenv
 
@@ -1106,7 +1105,7 @@ def create_dockerfile_buildbase(ddir, dockerfile_name, argmap):
       && . /etc/os-release \\
       && echo "deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ $UBUNTU_CODENAME main" | tee /etc/apt/sources.list.d/kitware.list >/dev/null \\
       && apt-get update -q=2 \\
-      && apt-get install -y --no-install-recommends cmake=3.27.7* cmake-data=3.27.7*
+      && apt-get install -y --no-install-recommends cmake=3.28.3* cmake-data=3.28.3*
 """
 
         if FLAGS.enable_gpu:
@@ -1243,7 +1242,6 @@ def create_dockerfile_linux(
     rm -fr ${TRT_ROOT}/bin ${TRT_ROOT}/targets/${ARCH}-linux-gnu/bin ${TRT_ROOT}/data && \\
     rm -fr ${TRT_ROOT}/doc ${TRT_ROOT}/onnx_graphsurgeon ${TRT_ROOT}/python && \\
     rm -fr ${TRT_ROOT}/samples ${TRT_ROOT}/targets/${ARCH}-linux-gnu/samples && \\
-    python3 -m pip install --upgrade pip && \\
     pip3 install --no-cache-dir transformers && \\
     find /usr -name libtensorrt_llm.so -exec dirname {} \; > /etc/ld.so.conf.d/tensorrt-llm.conf && \\
     find /opt/tritonserver -name libtritonserver.so -exec dirname {} \; > /etc/ld.so.conf.d/triton-tensorrtllm-worker.conf && \\
@@ -1308,6 +1306,7 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach
 # artifacts copied below are assign to this user.
 ENV TRITON_SERVER_USER=triton-server
 RUN userdel tensorrt-server > /dev/null 2>&1 || true \\
+      && userdel ubuntu > /dev/null 2>&1 || true \\
       && if ! id -u $TRITON_SERVER_USER > /dev/null 2>&1 ; then \\
           useradd $TRITON_SERVER_USER; \\
         fi \\
@@ -1354,12 +1353,10 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach
               libgoogle-perftools-dev \\
               libjemalloc-dev \\
               libnuma-dev \\
-              libre2-9 \\
               software-properties-common \\
               wget \\
               {backend_dependencies} \\
               python3-pip \\
-      && python3 -m pip install --upgrade pip \\
       && rm -rf /var/lib/apt/lists/*
 """.format(
             backend_dependencies=backend_dependencies
@@ -1402,7 +1399,8 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach
     if "python" in backends:
         if target_platform() == "rhel":
             df += """
-# python3 and some pip installs required for the python backend
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+# python3, python3-pip and some pip installs required for the python backend
 RUN yum install -y \\
         libarchive-devel \\
         openssl-devel \\
@@ -1420,16 +1418,17 @@ def dockerfile_prepare_container_linux(argmap, backends, enable_gpu, target_mach
 """
         else:
             df += """
-# python3 and some pip installs required for the python backend
+ENV PIP_BREAK_SYSTEM_PACKAGES=1
+# python3, python3-pip and some pip installs required for the python backend
 RUN apt-get update \\
       && apt-get install -y --no-install-recommends \\
             python3 \\
             libarchive-dev \\
+            python3-pip \\
+            python3-wheel \\
+            python3-setuptools \\
             libpython3-dev \\
-      && pip3 install --upgrade pip \\
       && pip3 install --upgrade \\
-            wheel \\
-            setuptools \\
             \"numpy<2\" \\
             virtualenv \\
       && rm -rf /var/lib/apt/lists/*
@@ -1631,7 +1630,7 @@ def create_build_dockerfiles(
             FLAGS.upstream_container_version
         )
     else:
-        base_image = "ubuntu:22.04"
+        base_image = "ubuntu:24.04"
 
     dockerfileargmap = {
         "NVIDIA_BUILD_REF": "" if FLAGS.build_sha is None else FLAGS.build_sha,

deploy/aws/values.yaml

@@ -27,7 +27,7 @@
 replicaCount: 1
 
 image:
-  imageName: nvcr.io/nvidia/tritonserver:24.10-py3
+  imageName: nvcr.io/nvidia/tritonserver:24.11-py3
   pullPolicy: IfNotPresent
   modelRepositoryPath: s3://triton-inference-server-repository/model_repository
   numGpus: 1

deploy/fleetcommand/values.yaml

@@ -27,7 +27,7 @@
 replicaCount: 1
 
 image:
-  imageName: nvcr.io/nvidia/tritonserver:24.10-py3
+  imageName: nvcr.io/nvidia/tritonserver:24.11-py3
   pullPolicy: IfNotPresent
   numGpus: 1
   serverCommand: tritonserver
@@ -47,13 +47,13 @@ image:
     #
     # To set model control mode, uncomment and configure below
     # TODO: Fix the following url, it is invalid
-    # See https://github.com/triton-inference-server/server/blob/r24.10/docs/model_management.md
+    # See https://github.com/triton-inference-server/server/blob/r24.11/docs/model_management.md
     #  for more details
     #- --model-control-mode=explicit|poll|none
     #
     # Additional server args
     #
-    # see https://github.com/triton-inference-server/server/blob/r24.10/README.md
+    # see https://github.com/triton-inference-server/server/blob/r24.11/README.md
     #  for more details
 
 service:

deploy/gcp/values.yaml

@@ -27,7 +27,7 @@
 replicaCount: 1
 
 image:
-  imageName: nvcr.io/nvidia/tritonserver:24.10-py3
+  imageName: nvcr.io/nvidia/tritonserver:24.11-py3
   pullPolicy: IfNotPresent
   modelRepositoryPath: gs://triton-inference-server-repository/model_repository
   numGpus: 1

deploy/gke-marketplace-app/benchmark/perf-analyzer-script/triton_client.yaml

@@ -33,7 +33,7 @@ metadata:
   namespace: default
 spec:
   containers:
-  - image: nvcr.io/nvidia/tritonserver:24.10-py3-sdk
+  - image: nvcr.io/nvidia/tritonserver:24.11-py3-sdk
     imagePullPolicy: Always
     name: nv-triton-client
     securityContext: