[DP][V1] Fix rank set in DP scenario

Signed-off-by: MengqingCao <cmq0113@163.com>

Author: MengqingCao

.github/workflows/accuracy_test.yaml

@@ -175,7 +175,7 @@ jobs:
         working-directory: ./vllm-ascend
         run: |
           pip install -r requirements-dev.txt
-          pip install -e .
+          pip install -e . --no-build-isolation
 
       - name: Install lm-eval, ray, and datasets
         run: |

.github/workflows/nightly_benchmarks.yaml

@@ -116,7 +116,7 @@ jobs:
 
       - name: Install vllm-project/vllm-ascend
         run: |
-          pip install -e .
+          pip install -e . --no-build-isolation
           pip install -r benchmarks/requirements-bench.txt
 
       - name: Run current commit benchmarks

.github/workflows/vllm_ascend_test.yaml

@@ -216,7 +216,7 @@ jobs:
       - name: Install vllm-project/vllm-ascend
         run: |
           pip install -r requirements-dev.txt
-          pip install -v -e .
+          pip install -v -e . --no-build-isolation
 
       - name: Run e2e test for V1 Engine
         env:
@@ -313,7 +313,7 @@ jobs:
       - name: Install vllm-project/vllm-ascend
         run: |
           pip install -r requirements-dev.txt
-          pip install -v -e .
+          pip install -v -e . --no-build-isolation
 
       - name: Run vllm-project/vllm-ascend test for V1 Engine
         env:

.github/workflows/vllm_ascend_test_long_term.yaml

@@ -90,7 +90,7 @@ jobs:
       - name: Install vllm-project/vllm-ascend
         run: |
           pip install -r requirements-dev.txt
-          pip install -v -e .
+          pip install -v -e . --no-build-isolation
 
       - name: Run vllm-project/vllm-ascend long term test
         run: |

.github/workflows/vllm_ascend_test_pd.yaml

@@ -99,7 +99,7 @@ jobs:
       - name: Install vllm-project/vllm-ascend
         run: |
           pip install -r requirements-dev.txt
-          pip install -v -e .
+          pip install -v -e . --no-build-isolation
 
       - name: Run vllm-project/vllm-ascend PD Disaggregation test
         run: |

Dockerfile

@@ -49,7 +49,7 @@ RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -v -e /vllm-workspace/vllm
 RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
     source /usr/local/Ascend/nnal/atb/set_env.sh && \
     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
+    python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --no-build-isolation --extra-index https://download.pytorch.org/whl/cpu/ && \
     python3 -m pip cache purge
 
 # Install modelscope (for fast download) and ray (for multinode)

Dockerfile.openEuler

@@ -46,7 +46,7 @@ RUN VLLM_TARGET_DEVICE="empty" python3 -m pip install -e /vllm-workspace/vllm/ -
 RUN source /usr/local/Ascend/ascend-toolkit/set_env.sh && \
     source /usr/local/Ascend/nnal/atb/set_env.sh && \
     export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/Ascend/ascend-toolkit/latest/`uname -i`-linux/devlib && \
-    python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --extra-index https://download.pytorch.org/whl/cpu/ && \
+    python3 -m pip install -v -e /vllm-workspace/vllm-ascend/ --no-build-isolation --extra-index https://download.pytorch.org/whl/cpu/ && \
     python3 -m pip cache purge
 
 # Install modelscope (for fast download) and ray (for multinode)

pyproject.toml

@@ -12,7 +12,7 @@ requires = [
     "scipy",
     "setuptools>=64",
     "setuptools-scm>=8",
-    "torch-npu==2.5.1",
+    "torch-npu==2.5.1.post1.dev20250528",
     "torch>=2.5.1",
     "torchvision<0.21.0",
     "wheel",

requirements.txt

@@ -10,7 +10,8 @@ pyyaml
 scipy
 setuptools>=64
 setuptools-scm>=8
-torch-npu==2.5.1
+--extra-index-url https://mirrors.huaweicloud.com/ascend/repos/pypi
+torch-npu==2.5.1.post1.dev20250528
 torch>=2.5.1
 torchvision<0.21.0
 wheel

tests/multicard/test_data_parallel.py

@@ -0,0 +1,66 @@
+#
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+# Copyright 2023 The vLLM team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+"""
+Compare the outputs of vLLM with and without aclgraph.
+
+Run `pytest tests/multicard/test_data_parallel.py`.
+"""
+
+import os
+
+import pytest
+
+from tests.conftest import VllmRunner
+from tests.model_utils import check_outputs_equal
+
+MODELS = ["Qwen/Qwen2.5-0.5B-Instruct"]
+
+
+@pytest.mark.skipif(os.getenv("VLLM_USE_V1") == "0",
+                    reason="Data parallel only support on v1")
+@pytest.mark.parametrize("model", MODELS)
+@pytest.mark.parametrize("max_tokens", [32])
+def test_data_parallel_correctness(
+    model: str,
+    max_tokens: int,
+) -> None:
+    example_prompts = [
+        "Hello, my name is", "The president of the United States is",
+        "The capital of France is", "The future of AI is"
+    ]
+
+    with VllmRunner(model_name=model,
+                    max_model_len=1024,
+                    max_num_seqs=16,
+                    data_parallel_size=2,
+                    distributed_executor_backend="mp") as vllm_model:
+        vllm_dp_outputs = vllm_model.generate_greedy(example_prompts,
+                                                     max_tokens)
+
+    with VllmRunner(
+            model_name=model,
+            max_model_len=1024,
+            max_num_seqs=16,
+    ) as vllm_model:
+        vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
+
+    check_outputs_equal(
+        outputs_0_lst=vllm_outputs,
+        outputs_1_lst=vllm_dp_outputs,
+        name_0="vllm_outputs",
+        name_1="vllm_dp_outputs",
+    )

vllm_ascend/patch/__init__.py

@@ -47,16 +47,7 @@
 #    Related PR (if no, explain why):
 #    Future Plan:
 #       Remove those patch when vllm merged them
-#   2. `vllm.v1.engine.core.DPEngineCoreProc._init_data_parallel`
-#    Why:
-#       There is some bug for ASCEND_RT_VISIBLE_DEVICES usage.
-#    How：
-#       The ASCEND_RT_VISIBLE_DEVICES related code is dropped.
-#    Related PR (if no, explain why):
-#       No, this is a bug for vllm ascend
-#    Future Plan:
-#       Remove this patch once ASCEND_RT_VISIBLE_DEVICES bug is fixed.
-#   3. `vllm.config.ParallelConfig.get_next_dp_init_port`
+#   2. `vllm.config.ParallelConfig.get_next_dp_init_port`
 #    Why:
 #       vllm doesn't support get port from environment.
 #    How：
@@ -65,7 +56,7 @@
 #       Need a PR to vllm to support get port from environment.
 #    Future Plan:
 #       Remove those patch when vllm merged them
-#   4. `vllm.config.ParallelConfig.ParallelConfig.stateless_init_dp_group`
+#   3. `vllm.config.ParallelConfig.ParallelConfig.stateless_init_dp_group`
 #    Why:
 #       vLLM use gloo backend by default to initialize stateless dp process gourp, but we want to use hccl here to
 #       get better performance

vllm_ascend/patch/platform/patch_common/patch_distributed.py

@@ -21,10 +21,9 @@
 import vllm.distributed
 import vllm.envs as envs
 from torch.distributed import ProcessGroup
-from vllm.config import ParallelConfig, VllmConfig
+from vllm.config import ParallelConfig
 from vllm.distributed.utils import \
     stateless_init_torch_distributed_process_group
-from vllm.v1.engine.core import DPEngineCoreProc
 
 
 def ascend_destroy_model_parallel():
@@ -79,21 +78,6 @@ def stateless_init_dp_group(self) -> "ProcessGroup":
     return dp_group
 
 
-def _init_data_parallel(self, vllm_config: VllmConfig):
-    # Configure NPUs and stateless process group for data parallel.
-    dp_rank = vllm_config.parallel_config.data_parallel_rank
-    dp_size = vllm_config.parallel_config.data_parallel_size
-    local_dp_rank = vllm_config.parallel_config.data_parallel_rank_local
-
-    assert dp_size > 1
-    assert 0 <= local_dp_rank <= dp_rank < dp_size
-
-    self.local_dp_rank = local_dp_rank
-    self.dp_group = vllm_config.parallel_config.stateless_init_dp_group()
-    self.current_wave = 0
-
-
 vllm.distributed.parallel_state.destroy_model_parallel = ascend_destroy_model_parallel
-DPEngineCoreProc._init_data_parallel = _init_data_parallel
 ParallelConfig.get_next_dp_init_port = parallel_config_get_dp_port
 ParallelConfig.stateless_init_dp_group = stateless_init_dp_group

vllm_ascend/worker/worker_v1.py

@@ -75,12 +75,6 @@ def __init__(
                          distributed_init_method=distributed_init_method,
                          is_driver_worker=is_driver_worker)
 
-        # NOTE(Yizhou): Since we do not set ASCEND_RT_VISIBLE_DEVICES in
-        # vllm_ascend, we need to set the device id manually.
-        local_dp_rank = self.vllm_config.parallel_config.data_parallel_rank_local
-        world_size = self.vllm_config.parallel_config.world_size
-        self.local_rank_across_dp = local_dp_rank * world_size + self.local_rank
-
         # Try to import mindie_turbo to accelerate vLLM inference.
         try_register_lib(
             "mindie_turbo",
@@ -124,7 +118,7 @@ def initialize_cache(self, num_gpu_blocks: int,
 
     def init_device(self):
         if self.device_config.device.type == "npu":
-            self.device = torch.device(f"npu:{self.local_rank_across_dp}")
+            self.device = torch.device(f"npu:{self.local_rank}")
             NPUPlatform.set_device(self.device)
             NPUPlatform.empty_cache()
             self.init_npu_memory = NPUPlatform.mem_get_info()[0]