Merge vl execution path into normal execution path (#2829)

* merge vl model into gpu_model runner

Change-Id: I9f4691a3d5f135e8d72b1d58abcd15ef3aa3f2a6

* fix chinese

Change-Id: Ic7405109b984c21e076fb3b01ff6feb571d0119a

* fix the parse parameter

Change-Id: I4cd62ee87c06220af580d91e347145d4394917fe

* fix the bug in online_inference

Change-Id: Idb111bb2114e83017c4050b2a68cf039c6d3c559

* polish code

Change-Id: I7d4194102c2f1b0743b74fbd5fc284eb8ef4d17c

Author: zeroRains

fastdeploy/config.py

@@ -18,7 +18,7 @@
 
 from dataclasses import dataclass, field
 from enum import Enum
-from typing import Literal, Optional, Union
+from typing import Literal, Optional
 
 from paddleformers.transformers.configuration_utils import PretrainedConfig
 from paddleformers.trl import llm_utils
@@ -89,6 +89,7 @@ def __init__(
         self.max_model_len = 0
         self.dtype = ""
         self.enable_logprob = False
+        self.enable_mm = False
 
         for key, value in args.items():
             if hasattr(self, key):

fastdeploy/engine/engine.py

@@ -990,8 +990,6 @@ def _start_worker_service(self):
         pd_cmd = pd_cmd + f" --log_dir {log_dir}"
 
         worker_path = "../worker/worker_process.py"
-        if self.cfg.enable_mm:
-            worker_path = "../worker/vl_worker_process.py"
         py_script = os.path.join(current_dir_path, worker_path)
 
         ori_vocab_size = (
@@ -1030,7 +1028,9 @@ def _start_worker_service(self):
             f" --speculative_benchmark_mode {self.cfg.speculative_config.benchmark_mode}"
             f" --max_capture_batch_size {self.cfg.max_capture_batch_size}"
             f" --guided_decoding_backend {self.cfg.guided_decoding_backend}"
-            f" --load_strategy {self.cfg.model_config.load_strategy}")
+            f" --load_strategy {self.cfg.model_config.load_strategy}"
+            f" --enable_mm {self.cfg.enable_mm}")
+
 
         worker_append_flag = {
             "enable_expert_parallel":

fastdeploy/model_executor/pre_and_post_process.py

@@ -129,6 +129,36 @@ def post_process_normal(sampler_output: SamplerOutput,
                         save_each_rank: bool = False,
                         skip_save_output: bool = False) -> ModelRunnerOutput:
     """ Post-processing steps after completing a single token generation. """
+    # handle vl:
+    if model_output.enable_thinking:
+        exists_think_end = sampler_output.sampled_token_ids == model_output.think_end_id
+        paddle.assign(
+                paddle.where(
+                    exists_think_end,
+                    model_output.need_think_end - 1,
+                    model_output.need_think_end,
+                ), model_output.need_think_end)
+
+        paddle.assign(
+            paddle.where(
+                model_output.need_think_end.cast("bool"),
+                model_output.reasoning_index - 1,
+                model_output.reasoning_index,
+            ), model_output.reasoning_index)
+
+        stop_wo_think = (
+            (sampler_output.sampled_token_ids == model_output.eos_token_id) |
+            (model_output.reasoning_index == 0)) & (
+                model_output.need_think_end > 0)
+        sampler_output.sampled_token_ids = paddle.where(stop_wo_think,
+                                    model_output.think_end_id,
+                                    sampler_output.sampled_token_ids)
+        paddle.assign(
+            paddle.where(
+                stop_wo_think,
+                model_output.need_think_end - 1,
+                model_output.need_think_end,
+            ), model_output.need_think_end)
     # 1. Set stop value
     paddle.assign(
         paddle.where(

fastdeploy/worker/gpu_model_runner.py

@@ -201,6 +201,27 @@ class ModelOutputData:
     """
     accept_num: paddle.Tensor
 
+    """
+        vl model enable to think
+    """
+    enable_thinking: paddle.Tensor = None
+
+    """
+        vl model think end id
+    """
+    think_end_id: int = -1
+
+    """
+        vl model need to think
+    """
+    need_think_end: paddle.Tensor = None
+
+    """
+        vl model reasoning index
+    """
+    reasoning_index: paddle.Tensor = None
+
+
 
 @dataclass
 class ModelRunnerOutput: