From 8e90c7d683a2e92730fb9941fd088d46127a544e Mon Sep 17 00:00:00 2001 From: Jackmin801 Date: Sat, 21 Jun 2025 13:09:41 -0700 Subject: [PATCH] load existing generator if it exists. this happens in preemption --- vllm/model_executor/sampling_metadata.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/sampling_metadata.py b/vllm/model_executor/sampling_metadata.py index 56f0f0984bfa..24df546c6a11 100644 --- a/vllm/model_executor/sampling_metadata.py +++ b/vllm/model_executor/sampling_metadata.py @@ -273,7 +273,9 @@ def _prepare_seq_groups( do_sample = seq_group_metadata.do_sample if seq_group_metadata.is_prompt: - if sampling_params.seed is not None: + if seq_group_metadata.request_id in generators: + generator = generators[seq_group_metadata.request_id] + elif sampling_params.seed is not None: generator = torch.Generator(device=device).manual_seed( sampling_params.seed) if generators is not None: