[CI] Sync test dependency with test.in for torch nightly (#19632)

Signed-off-by: Yang Wang <elainewy@meta.com>
Signed-off-by: Yida Wu <yidawu@alumni.cmu.edu>
Signed-off-by: Nick Hill <nhill@redhat.com>
Co-authored-by: Concurrensee <yida.wu@amd.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Co-authored-by: Nick Hill <nhill@redhat.com>

Author: 5 people

.buildkite/test-pipeline.yaml

@@ -41,6 +41,16 @@ steps:
   # TODO: add `--strict` once warnings in docstrings are fixed
   - mkdocs build
 
+- label: Pytorch Nightly Dependency Override Check # 2min
+  # if this test fails, it means the nightly torch version is not compatible with some
+  # of the dependencies. Please check the error message and add the package to whitelist
+  # in /vllm/tools/generate_nightly_torch_test.py
+  soft_fail: true
+  source_file_dependencies:
+  - requirements/nightly_torch_test.txt
+  commands:
+  - bash standalone_tests/pytorch_nightly_dependency.sh
+
 - label: Async Engine, Inputs, Utils, Worker Test # 24min
   mirror_hardwares: [amdexperimental]
   source_file_dependencies:
@@ -767,7 +777,7 @@ steps:
     - bash weight_loading/run_model_weight_loading_test.sh -c weight_loading/models.txt
 
 - label: Weight Loading Multiple GPU Test - Large Models # optional
-  mirror_hardwares: [amdexperimental] 
+  mirror_hardwares: [amdexperimental]
   working_dir: "/vllm-workspace/tests"
   num_gpus: 2
   gpu: a100

.pre-commit-config.yaml

@@ -53,6 +53,11 @@ repos:
       files: ^requirements/test\.(in|txt)$
 - repo: local
   hooks:
+  - id: format-torch-nightly-test
+    name: reformat nightly_torch_test.txt to be in sync with test.in
+    language: python
+    entry: python tools/generate_nightly_torch_test.py
+    files: ^requirements/test\.(in|txt)$
   - id: mypy-local
     name: Run mypy for local Python installation
     entry: tools/mypy.sh 0 "local"

requirements/nightly_torch_test.txt

@@ -1,47 +1,50 @@
-# Dependency that able to run entrypoints test
-# pytest and its extensions
+# testing
 pytest
-pytest-asyncio
+tensorizer>=2.9.0
 pytest-forked
-pytest-mock
+pytest-asyncio
 pytest-rerunfailures
 pytest-shard
 pytest-timeout
 
-librosa # required by audio tests in entrypoints/openai
-sentence-transformers # required for embedding tests
-transformers==4.52.4
-transformers_stream_generator # required for qwen-vl test
-numba == 0.61.2; python_version > '3.9'
 # testing utils
-boto3
-botocore
-datasets
-ray >= 2.10.0
+backoff # required for phi4mm test
+blobfile # required for kimi-vl test
+einops # required for MPT, qwen-vl and Mamba
+httpx
+librosa # required for audio tests
+vocos # required for minicpmo_26 test
 peft
-runai-model-streamer==0.11.0
-runai-model-streamer-s3==0.11.0
-tensorizer>=2.9.0
-lm-eval==0.4.8
-buildkite-test-collector==0.1.9
+pqdm
+ray[cgraph,default]>=2.43.0, !=2.44.* # Ray Compiled Graph, required by pipeline parallelism tests
+sentence-transformers # required for embedding tests
+soundfile # required for audio tests
+jiwer # required for audio tests
+timm # required for internvl test
+transformers_stream_generator # required for qwen-vl test
+matplotlib # required for qwen-vl test
+mistral_common[opencv] >= 1.6.2 # required for pixtral test
+num2words # required for smolvlm test
+opencv-python-headless >= 4.11.0 # required for video test
+datamodel_code_generator # required for minicpm3 test
 lm-eval[api]==0.4.8 # required for model evaluation test
-
-# required for quantization test
+mteb>=1.38.11, <2 # required for mteb test
+transformers==4.52.4
+tokenizers==0.21.1
+huggingface-hub[hf_xet]>=0.30.0  # Required for Xet downloads.
+schemathesis>=3.39.15 # Required for openai schema test.
+# quantization
 bitsandbytes>=0.45.3
+buildkite-test-collector==0.1.9
 
-# required for minicpmo_26 test
-vector_quantize_pytorch
-vocos
-
-# required for Basic Models Test
-blobfile # required for kimi-vl test
-matplotlib # required for qwen-vl test
 
-# required for  Multi-Modal Models Test (Standard)
-num2words # required for smolvlm test
-pqdm
-timm # required for internvl test
-mistral-common==1.6.2
+genai_perf==0.0.8
+tritonclient==2.51.0
 
-schemathesis==3.39.15  # Required for openai schema test.
-mteb>=1.38.11, <2 # required for mteb test
+numba == 0.60.0; python_version == '3.9' # v0.61 doesn't support Python 3.9. Required for N-gram speculative decoding
+numba == 0.61.2; python_version > '3.9'
+numpy
+runai-model-streamer==0.11.0
+runai-model-streamer-s3==0.11.0
+fastsafetensors>=0.1.10
+pydantic>=2.10 # 2.9 leads to error on python 3.10

requirements/test.in

@@ -42,6 +42,7 @@ schemathesis>=3.39.15 # Required for openai schema test.
 bitsandbytes>=0.45.3
 buildkite-test-collector==0.1.9
 
+
 genai_perf==0.0.8
 tritonclient==2.51.0
 
@@ -51,4 +52,4 @@ numpy
 runai-model-streamer==0.11.0
 runai-model-streamer-s3==0.11.0
 fastsafetensors>=0.1.10
-pydantic>=2.10 # 2.9 leads to error on python 3.10
+pydantic>=2.10 # 2.9 leads to error on python 3.10

tests/standalone_tests/pytorch_nightly_dependency.sh

@@ -0,0 +1,42 @@
+#!/bin/sh
+# This script tests if the nightly torch packages are not overridden by the dependencies
+
+set -e
+set -x
+
+cd /vllm-workspace/
+
+rm -rf .venv
+
+uv venv .venv
+
+source .venv/bin/activate
+
+# check the environment
+uv pip freeze
+
+echo ">>> Installing nightly torch packages"
+uv pip install --quiet torch torchvision torchaudio --pre --extra-index-url https://download.pytorch.org/whl/nightly/cu128
+
+echo ">>> Capturing torch-related versions before requirements install"
+uv pip freeze | grep -E '^torch|^torchvision|^torchaudio' | sort > before.txt
+echo "Before:"
+cat before.txt
+
+echo ">>> Installing requirements/nightly_torch_test.txt"
+uv pip install --quiet -r requirements/nightly_torch_test.txt
+
+echo ">>> Capturing torch-related versions after requirements install"
+uv pip freeze | grep -E '^torch|^torchvision|^torchaudio' | sort > after.txt
+echo "After:"
+cat after.txt
+
+echo ">>> Comparing versions"
+if diff before.txt after.txt; then
+  echo "torch version not overridden."
+else
+  echo "torch version overridden by nightly_torch_test.txt, \
+  if the dependency is not triggered by the pytroch nightly test,\
+  please add the dependency to the list 'white_list'  in tools/generate_nightly_torch_test.py"
+  exit 1
+fi

tools/generate_nightly_torch_test.py

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Generates specialized requirements files for nightly PyTorch testing.
+
+This script reads the main test requirements input file (`requirements/test.in`)
+and splits its content into two files:
+1.  `requirements/nightly_torch_test.txt`: Contains dependencies
+except PyTorch-related.
+2.  `torch_nightly_test.txt`: Contains only PyTorch-related packages.
+"""
+
+input_file = "requirements/test.in"
+output_file = "requirements/nightly_torch_test.txt"
+
+# white list of packages that are not compatible with PyTorch nightly directly
+# with pip install. Please add your package to this list if it is not compatible
+# or make the dependency test fails.
+white_list = ["torch", "torchaudio", "torchvision", "mamba_ssm"]
+
+with open(input_file) as f:
+    lines = f.readlines()
+
+skip_next = False
+
+for line in lines:
+    if skip_next:
+        if line.startswith((" ", "\t")) or line.strip() == "":
+            continue
+        skip_next = False
+
+    if any(k in line.lower() for k in white_list):
+        skip_next = True
+        continue