File tree Expand file tree Collapse file tree 1 file changed +8
-3
lines changed Expand file tree Collapse file tree 1 file changed +8
-3
lines changed Original file line number Diff line number Diff line change @@ -1120,14 +1120,19 @@ def _process_reqs(
1120
1120
if self .dp_size > 1 else num_input_tokens )
1121
1121
padded_batch_size = self .select_torchair_padded_batch_size (
1122
1122
max_num_tokens )
1123
- num_tokens_across_dp .masked_fill_ (num_tokens_across_dp == - 1 ,
1124
- padded_batch_size )
1125
1123
graph_pad_size = padded_batch_size - total_num_scheduled_tokens
1126
1124
extra_builder_kwargs ['graph_pad_size' ] = graph_pad_size
1125
+ # If torchair graph is enabled and in decode mode, the dummy run
1126
+ # batch size is set to the selected graph size.
1127
+ dummy_num_tokens = padded_batch_size
1127
1128
else :
1128
1129
# If torchair graph is not enabled, or if with_prefill is True, the
1129
1130
# dummy run batch size is set to 1.
1130
- num_tokens_across_dp .masked_fill_ (num_tokens_across_dp == - 1 , 1 )
1131
+ dummy_num_tokens = 1
1132
+
1133
+ if self .dp_size > 1 :
1134
+ num_tokens_across_dp .masked_fill_ (num_tokens_across_dp == - 1 ,
1135
+ dummy_num_tokens )
1131
1136
1132
1137
if self .vllm_config .model_config .use_mla :
1133
1138
attn_metadata = self .attn_metadata_builder .build ( # type: ignore
You can’t perform that action at this time.
0 commit comments