Merge remote-tracking branch 'upstream/aice/v1.21.0' into gyou/aice/v1.21.0/qwen3

Author: gyou2021

vllm/model_executor/layers/pooler.py

@@ -241,6 +241,7 @@ def forward(self, pooled_data: Union[list[torch.Tensor], torch.Tensor],
         dimensions_list = [
             pooling_param.dimensions
             for _, pooling_param in pooling_metadata.seq_groups
+            if pooling_param is not None
         ]
         if any(d is not None for d in dimensions_list):
             # change the output dimension

vllm/model_executor/layers/vocab_parallel_embedding.py

@@ -395,7 +395,7 @@ def weight_loader(self, param: Parameter, loaded_weight: torch.Tensor):
                 padded_weight = torch.cat([
                     loaded_weight,
                     torch.zeros(param.shape[0] - loaded_weight.shape[0],
-                                *loaded_weight.shape[1:])
+                                *loaded_weight.shape[1:], device=loaded_weight.device)
                 ])
             else:
                 padded_weight = loaded_weight

vllm/model_executor/models/qwen_vl.py

@@ -774,7 +774,7 @@ def get_input_embeddings(
         inputs_embeds = self.transformer.get_input_embeddings(input_ids)
 
         if multimodal_embeddings is not None:
-            inputs_embeds = self._merge_multimodal_embeddings(
+            inputs_embeds = merge_multimodal_embeddings(
                 input_ids, inputs_embeds, multimodal_embeddings,
                 self.transformer.visual.image_pad_id)
 

vllm/model_executor/models/utils.py

@@ -343,7 +343,6 @@ def _flatten_embeddings(embeddings: NestedTensors) -> torch.Tensor:
     if isinstance(embeddings, torch.Tensor):
         # Flatten all but the last dimension.
         return embeddings.flatten(0, -2)
-
     return torch.cat(tuple(_flatten_embeddings(t) for t in embeddings))
 
 
@@ -391,8 +390,19 @@ def _merge_multimodal_embeddings(
     """
     # skip check for HPU, the number of tokens is a cpu fallback during HPU lazy
     if current_platform.is_hpu():
-        flattened = _flatten_embeddings(multimodal_embeddings)
-        inputs_embeds[is_multimodal] = flattened
+
+        if isinstance(multimodal_embeddings, torch.Tensor):
+            is_multimodal = is_multimodal.reshape(-1)
+            batch_size, seq_length, hidden_size = inputs_embeds.shape
+            inputs_embeds = inputs_embeds.reshape(-1, hidden_size)
+            flattened = multimodal_embeddings.reshape(-1, hidden_size)
+            inputs_embeds[is_multimodal] = flattened
+            inputs_embeds = inputs_embeds.reshape(batch_size, seq_length,
+                                              hidden_size)
+        else:
+            flattened = _flatten_embeddings(multimodal_embeddings)
+            inputs_embeds[is_multimodal] = flattened   
+               
         return inputs_embeds
 
     num_expected_tokens = is_multimodal.sum().item()
@@ -492,7 +502,6 @@ def merge_multimodal_embeddings(
             torch.isin(input_ids, placeholder_token_id),
             multimodal_embeddings,
         )
-
     return _merge_multimodal_embeddings(
         inputs_embeds,
         (input_ids == placeholder_token_id),
@@ -712,7 +721,6 @@ def extract_layer_index(layer_name: str) -> int:
                                 " only contain one integer")
     return int_vals[0]
 
-
 def get_input_mask(hidden_states: torch.Tensor,
                    valid_len: torch.Tensor) -> torch.Tensor:
     """
@@ -727,7 +735,6 @@ def get_input_mask(hidden_states: torch.Tensor,
     mask = mask.to(hidden_states.dtype)
     return mask
 
-
 def cast_overflow_tensors(
     tensors: torch.Tensor,
     offset: float = 1000,

vllm/worker/hpu_model_runner.py

@@ -396,6 +396,8 @@ def _set_attn_bias(self, attn_metadata, batch_size, seq_len, device,
             len_mask_v = len_mask.view(batch_size, 1, seq_len, 1)
             mask = attn_mask.logical_or(len_mask).logical_or(len_mask_v)
             off_value = -3E38  #small number, avoid nan and overflow
+            if dtype == torch.float16:
+                off_value = -63000    # a small value close to float16.min
         else:
             mask = attn_mask.logical_or(
                 len_mask)  #no need for len_mask_v as decode overwrites it