[vl]remove duplicated load logic (#2744)

Author: bukejiyu

fastdeploy/config.py

@@ -18,7 +18,7 @@
 
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Literal, Optional
+from typing import Literal, Optional, Union
 
 from paddleformers.transformers.configuration_utils import PretrainedConfig
 
@@ -72,8 +72,10 @@ def __init__(
         rope_theta: int = 10000,
         rope_3d: bool = False,
         ori_vocab_size: int | None = None,
-        moe_layer_start_index: int | None = None,
-        moe_layer_end_index: int | None = None,
+        moe_layer_start_index: Union[int, list[int], None] = None,
+        moe_num_experts: Union[int, list[int], None] = None,
+        moe_layer_end_index: Union[int, list[int], None] = None,
+        moe_num_shared_experts: int | None = None,
         num_hidden_layers: int | None = None,
         prefix_name="",
         freeze_embedding=False,
@@ -110,14 +112,10 @@ def __init__(
         self.prefix_name = prefix_name
         self.freeze_embedding = freeze_embedding
         self.rope_head_dim = rope_head_dim
-        moe_num_experts = kwargs.get("moe_num_experts", 0)
-        if moe_layer_start_index is not None:
-            self.moe_layer_start_index = moe_layer_start_index
-        elif moe_num_experts == 0:
-            self.moe_layer_start_index = self.num_layers
-            self.moe_num_experts = 0
-        if moe_layer_end_index is not None:
-            self.moe_layer_end_index = moe_layer_end_index
+        self.moe_layer_start_index = moe_layer_start_index
+        self.moe_num_experts = moe_num_experts
+        self.moe_num_shared_experts = moe_num_shared_experts
+        self.moe_layer_end_index = moe_layer_end_index
         self.ffn_hidden_size = ffn_hidden_size
         self.rope_3d = rope_3d
         self.start_layer_index = start_layer_index
@@ -132,15 +130,15 @@ class MoEConfig:
     """
     Configuration for MoE.
     """
-    num_experts: int = -1
+    num_experts: Union[int, list[int], None] = None
     top_k: int = 8
     moe_intermediate_size: int = -1
     num_experts_per_rank: int = -1
     num_experts_start_offset: int = -1
 
     moe_num_shared_experts = (0, )
-    moe_layer_start_index = 0
-    moe_layer_end_index = None
+    moe_layer_start_index: Union[int, list[int], None] = None
+    moe_layer_end_index: Union[int, list[int], None] = None
     moe_use_aux_free: bool = False
     num_max_dispatch_tokens_per_rank = 256
     im_patch_id = (

fastdeploy/model_executor/model_loader.py

@@ -28,6 +28,8 @@
     Ernie4_5_PretrainedModel
 from fastdeploy.model_executor.models.ernie4_5_mtp import \
     Ernie4_5_MTPPretrainedModel
+from fastdeploy.model_executor.models.ernie4_5_vl.ernie4_5_vl_moe import \
+    Ernie4_5_VLPretrainedModel
 from fastdeploy.model_executor.models.model_base import ModelRegistry
 from fastdeploy.model_executor.models.qwen2 import Qwen2PretrainedModel
 from fastdeploy.model_executor.models.qwen3 import Qwen3PretrainedModel
@@ -42,6 +44,7 @@
     "Qwen3MoeForCausalLM": Qwen3MoePretrainedModel,
     "Ernie4_5_ForCausalLM": Ernie4_5_PretrainedModel,
     "DeepseekV3ForCausalLM": DeepSeekV3PretrainedModel,
+    "Ernie4_5_VLMoeForConditionalGeneration": Ernie4_5_VLPretrainedModel,
 }
 
 
@@ -94,7 +97,7 @@ def load_model(self, fd_config: FDConfig) -> nn.Layer:
 
         if fd_config.load_config.dynamic_load_weight:
             # register rl model
-            import fastdeploy.rl
+            import fastdeploy.rl  # noqa
             architectures = architectures + "RL"
 
         with context:

fastdeploy/model_executor/models/ernie4_5_moe.py

@@ -26,6 +26,7 @@
 from paddleformers.utils.log import logger
 
 from fastdeploy.config import FDConfig, ModelConfig
+from fastdeploy.model_executor.forward_meta import ForwardMeta
 from fastdeploy.model_executor.graph_optimization.decorator import \
     support_graph_optimization
 from fastdeploy.model_executor.layers.activation import SiluAndMul
@@ -41,7 +42,6 @@
 from fastdeploy.model_executor.models.utils import \
     LayerIdPlaceholder as layerid
 from fastdeploy.model_executor.models.utils import WeightMeta
-from fastdeploy.model_executor.forward_meta import ForwardMeta
 
 
 class Ernie4_5_MLP(nn.Layer):
@@ -599,27 +599,13 @@ def get_tensor_parallel_split_mappings(num_layers, moe_num_experts,
             start_layer = (moe_layer_start_index
                            if moe_layer_start_index > 0 else num_layers)
             final_actions = build_expanded_keys(
-                num_layers,
-                moe_num_experts,
-                start_layer,
-                base_actions,
+                base_actions, num_layers, start_layer, moe_num_experts
             )
             return final_actions
-
-        moe_num_experts = 0
-        if isinstance(config.moe_num_experts, list):
-            moe_num_experts = sum(config.moe_num_experts)
-        elif isinstance(config.moe_num_experts, int):
-            moe_num_experts = config.moe_num_experts
-
-        moe_layer_start_index = -1
-        if isinstance(config.moe_layer_start_index, list):
-            moe_layer_start_index = min(config.moe_layer_start_index)
-        elif isinstance(config.moe_layer_start_index, int):
-            moe_layer_start_index = config.moe_layer_start_index
-
-        mappings = get_tensor_parallel_split_mappings(config.num_layers,
-                                                      moe_num_experts,
-                                                      moe_layer_start_index,
-                                                      config.prefix_name)
+        mappings = get_tensor_parallel_split_mappings(
+            config.num_layers,
+            config.moe_num_experts,
+            config.moe_layer_start_index,
+            config.prefix_name,
+        )
         return mappings

fastdeploy/model_executor/models/ernie4_5_vl/dfnrope/modeling.py

@@ -29,6 +29,8 @@
     flash_attn_unpadded as flash_attn_varlen_func
 from paddleformers.transformers.model_utils import PretrainedModel
 
+from fastdeploy.model_executor.layers.utils import get_tensor
+
 from .activation import ACT2FN
 from .configuration import DFNRopeVisionTransformerConfig
 
@@ -487,10 +489,10 @@ class DFNRopeVisionTransformerPretrainedModel(PretrainedModel):
 
     config_class = DFNRopeVisionTransformerConfig
 
-    def __init__(self, config) -> None:
+    def __init__(self, config, prefix_name: str = "") -> None:
         super().__init__(config)
         self.spatial_merge_size = config.spatial_merge_size
-
+        self.prefix_name = prefix_name
         self.patch_embed = PatchEmbed(
             patch_size=config.patch_size,
             in_channels=config.in_channels,
@@ -723,10 +725,18 @@ def get_tensor_parallel_split_mappings(depth):
         mappings = get_tensor_parallel_split_mappings(vision_config.depth)
         return mappings
 
-    def set_state_dict(self, state_dict, *args, **kwargs):
-        """_summary_
-
-        Args:
-            state_dict (_type_): _description_
-        """
-        super().set_state_dict(state_dict, *args, **kwargs)
+    def load_state_dict(self, state_dict):
+        params_dict = dict(self.named_parameters())
+        for param_name, param in params_dict.items():
+            state_dict_key = f"{self.prefix_name}.{param_name}"
+            if state_dict_key not in state_dict:
+                raise ValueError(
+                    f"The key {state_dict_key} does not exist in state_dict. "
+                )
+            tensor = get_tensor(state_dict.pop(state_dict_key))
+            if param.shape != tensor.shape:
+                raise ValueError(
+                    f"{state_dict_key} param.shape={param.shape} tensor.shape={tensor.shape}"
+                )
+            else:
+                param.copy_(tensor, False)