feat:add_glm_4_0414;fix openai client create bugs (#104)

11zhouxuan · web-flow · commit 4ac85464ab3c · 2025-04-24T09:31:38.000+08:00
* merge

* merge

* add Mistral-Small-3.1-24B-Instruct-2503

* modify qwq-32b deploy

* add txgemma model;

* modify model list command

* fix typo

* add some ecs parameters

* add glm4-z1 models

* modify vllm backend
diff --git a/src/emd/commands/deploy.py b/src/emd/commands/deploy.py
@@ -239,6 +239,9 @@ def deploy(
     dockerfile_local_path: Annotated[
         str, typer.Option("--dockerfile-local-path", help="Your custom Dockerfile path for building the model image, all files must be in the same directory")
     ] = None,
+    local_gpus:Annotated[
+        str, typer.Option("--local-gpus", help="Local gpu ids to deploy the model (e.g. `0,1,2`), only working with local deployment mode.")
+    ] = None,
 ):
     if only_allow_local_deploy:
         allow_local_deploy = True
@@ -389,8 +392,10 @@ def deploy(
     )
     if service_type == ServiceType.LOCAL:
         if check_cuda_exists():
-            if os.environ.get('CUDA_VISIBLE_DEVICES'):
-                console.print(f"[bold blue]local gpus: {os.environ.get('CUDA_VISIBLE_DEVICES')}[/bold blue]")
+            if local_gpus is not None:
+                os.environ['CUDA_VISIBLE_DEVICES']=local_gpus
+            elif os.environ.get('CUDA_VISIBLE_DEVICES'):
+                pass
             else:
                 gpu_num = get_gpu_num()
                 support_gpu_num = model.supported_instances[0].gpu_num
@@ -400,6 +405,7 @@ def deploy(
                         default=f"{default_gpus_str}"
                     ).ask()
                 os.environ['CUDA_VISIBLE_DEVICES']=gpus_to_deploy
+            console.print(f"[bold blue]local gpus: {os.environ.get('CUDA_VISIBLE_DEVICES')}[/bold blue]")
         instance_type = InstanceType.LOCAL
     else:
         if instance_type is None:
diff --git a/src/emd/models/engines.py b/src/emd/models/engines.py
@@ -165,6 +165,25 @@ class KtransformersEngine(OpenAICompitableEngine):
 
 vllm_glm4_engine064 = vllm_engine064
 
+
+vllm_glm4_0414_engine082 = VllmEngine(**{
+            **vllm_qwen25vl72b_engine073.model_dump(),
+            "engine_dockerfile_config": {"VERSION":"glm_z1_and_0414"},
+            "environment_variables": "export VLLM_USE_V1=0 && export VLLM_ATTENTION_BACKEND=FLASHINFER && export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
+            # "default_cli_args": "--max_model_len 16000  --max_num_seq 10 --disable-log-stats --enable-auto-tool-choice --tool-call-parser pythonic"
+            "default_cli_args": "--max_model_len 16000  --max_num_seq 10 --disable-log-stats"
+})
+
+
+vllm_glm4_z1_engine082 = VllmEngine(**{
+            **vllm_qwen25vl72b_engine073.model_dump(),
+            "engine_dockerfile_config": {"VERSION":"glm_z1_and_0414"},
+            "environment_variables": "export VLLM_USE_V1=0 && export VLLM_ATTENTION_BACKEND=FLASHINFER && export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True",
+            # "default_cli_args": "--max_model_len 16000  --max_num_seq 10 --disable-log-stats --enable-auto-tool-choice --tool-call-parser pythonic --enable-reasoning --reasoning-parser granite"
+            "default_cli_args": "--max_model_len 16000  --max_num_seq 10 --disable-log-stats --enable-reasoning --reasoning-parser granite"
+})
+
+
 vllm_glm4_wo_flashinfer_engine064 = VllmEngine(**{
              **vllm_engine064.model_dump(),
             #  "engine_dockerfile_config": {"VERSION":"v0.6.0"},
diff --git a/src/emd/models/llms/glm.py b/src/emd/models/llms/glm.py
@@ -1,5 +1,10 @@
 from .. import Model
-from ..engines import vllm_glm4_engine064,vllm_glm4_wo_flashinfer_engine064
+from ..engines import (
+    vllm_glm4_engine064,
+    vllm_glm4_wo_flashinfer_engine064,
+    vllm_glm4_0414_engine082,
+    vllm_glm4_z1_engine082
+)
 from ..services import (
     sagemaker_service,
     sagemaker_async_service,
@@ -79,3 +84,148 @@
         model_series=GLM4_SERIES
     )
 )
+
+
+Model.register(
+    dict(
+        model_id = "GLM-4-9B-0414",
+        supported_engines=[vllm_glm4_0414_engine082],
+        supported_instances=[
+            g5d12xlarge_instance,
+            g5d24xlarge_instance,
+            g5d48xlarge_instance,
+            local_instance
+        ],
+        supported_services=[
+            sagemaker_service,
+            sagemaker_async_service,
+            ecs_service,
+            local_service
+        ],
+        allow_china_region=True,
+        supported_frameworks=[fastapi_framework],
+        huggingface_model_id="THUDM/GLM-4-9B-0414",
+        modelscope_model_id="ZhipuAI/GLM-4-9B-0414",
+        require_huggingface_token=False,
+        application_scenario="Agent, tool use, translation, summary",
+        description="GLM-4-32B-0414 series",
+        model_type=ModelType.LLM,
+        model_series=GLM4_SERIES
+    )
+)
+
+Model.register(
+    dict(
+        model_id = "GLM-4-32B-0414",
+        supported_engines=[vllm_glm4_0414_engine082],
+        supported_instances=[
+            g5d12xlarge_instance,
+            g5d24xlarge_instance,
+            g5d48xlarge_instance,
+            local_instance
+        ],
+        supported_services=[
+            sagemaker_service,
+            sagemaker_async_service,
+            ecs_service,
+            local_service
+        ],
+        allow_china_region=True,
+        supported_frameworks=[fastapi_framework],
+        huggingface_model_id="THUDM/GLM-4-32B-0414",
+        modelscope_model_id="ZhipuAI/GLM-4-32B-0414",
+        require_huggingface_token=False,
+        application_scenario="Agent, tool use, translation, summary",
+        description="GLM-4-32B-0414 series",
+        model_type=ModelType.LLM,
+        model_series=GLM4_SERIES
+    )
+)
+
+
+
+Model.register(
+    dict(
+        model_id = "GLM-Z1-9B-0414",
+        supported_engines=[vllm_glm4_z1_engine082],
+        supported_instances=[
+            g5d12xlarge_instance,
+            g5d24xlarge_instance,
+            g5d48xlarge_instance,
+            local_instance
+        ],
+        supported_services=[
+            sagemaker_service,
+            sagemaker_async_service,
+            ecs_service,
+            local_service
+        ],
+        allow_china_region=True,
+        supported_frameworks=[fastapi_framework],
+        huggingface_model_id="THUDM/GLM-Z1-9B-0414",
+        modelscope_model_id="ZhipuAI/GLM-Z1-9B-0414",
+        require_huggingface_token=False,
+        application_scenario="Agent, tool use, translation, summary",
+        description="GLM-4-32B-0414 series",
+        model_type=ModelType.LLM,
+        model_series=GLM4_SERIES
+    )
+)
+
+
+Model.register(
+    dict(
+        model_id = "GLM-Z1-32B-0414",
+        supported_engines=[vllm_glm4_z1_engine082],
+        supported_instances=[
+            g5d12xlarge_instance,
+            g5d24xlarge_instance,
+            g5d48xlarge_instance,
+            local_instance
+        ],
+        supported_services=[
+            sagemaker_service,
+            sagemaker_async_service,
+            ecs_service,
+            local_service
+        ],
+        allow_china_region=True,
+        supported_frameworks=[fastapi_framework],
+        huggingface_model_id="THUDM/GLM-Z1-32B-0414",
+        modelscope_model_id="ZhipuAI/GLM-Z1-32B-0414",
+        require_huggingface_token=False,
+        application_scenario="Agent, tool use, translation, summary",
+        description="GLM-4-32B-0414 series",
+        model_type=ModelType.LLM,
+        model_series=GLM4_SERIES
+    )
+)
+
+
+Model.register(
+    dict(
+        model_id = "GLM-Z1-Rumination-32B-0414",
+        supported_engines=[vllm_glm4_z1_engine082],
+        supported_instances=[
+            g5d12xlarge_instance,
+            g5d24xlarge_instance,
+            g5d48xlarge_instance,
+            local_instance
+        ],
+        supported_services=[
+            sagemaker_service,
+            sagemaker_async_service,
+            ecs_service,
+            local_service
+        ],
+        allow_china_region=True,
+        supported_frameworks=[fastapi_framework],
+        huggingface_model_id="THUDM/GLM-Z1-Rumination-32B-0414",
+        modelscope_model_id="ZhipuAI/GLM-Z1-Rumination-32B-0414",
+        require_huggingface_token=False,
+        application_scenario="Agent, tool use, translation, summary",
+        description="GLM-4-32B-0414 series",
+        model_type=ModelType.LLM,
+        model_series=GLM4_SERIES
+    )
+)
diff --git a/src/pipeline/backend/convert_engine_image_to_dmaa_dockerfiles/zhipu_z1_vllm_image_dockerfile b/src/pipeline/backend/convert_engine_image_to_dmaa_dockerfiles/zhipu_z1_vllm_image_dockerfile
@@ -0,0 +1,8 @@
+From vllm/vllm-openai:v0.8.4
+
+RUN git clone https://github.com/vllm-project/vllm.git && cd vllm && git fetch origin pull/16618/head:pr-16618 && VLLM_USE_PRECOMPILED=1 pip install --editable .
+
+EXPOSE 8080
+
+# Set the serve script as the entrypoint
+ENTRYPOINT ["/usr/bin/serve"]
diff --git a/src/pipeline/backend/vllm/vllm_backend.py b/src/pipeline/backend/vllm/vllm_backend.py
@@ -2,7 +2,7 @@
 import sys
 import os
 from emd.models.utils.constants import ModelType
-
+import inspect
 from backend.backend import OpenAICompitableProxyBackendBase
 from emd.utils.logger_utils import get_logger
 
@@ -22,6 +22,13 @@ def create_proxy_server_start_command(self,model_path):
             serve_command += f" --api-key {self.api_key}"
         return serve_command
 
+    def openai_create_helper(self,fn:callable,request:dict):
+        sig = inspect.signature(fn)
+        extra_body = request.get("extra_body",{})
+        extra_params = {k:request.pop(k) for k in list(request.keys()) if k not in sig.parameters}
+        extra_body.update(extra_params)
+        request['extra_body'] = extra_body
+        return fn(**request)
 
     def invoke(self, request):
         # Transform input to vllm format
@@ -30,7 +37,7 @@ def invoke(self, request):
         logger.info(f"Chat request:{request}")
         if self.model_type == ModelType.EMBEDDING:
             # print('cal embedding....')
-            response = self.client.embeddings.create(**request)
+            response =self.openai_create_helper(self.client.embeddings.create,request)
             # print('end cal embedding....')
         elif self.model_type == ModelType.RERANK:
             headers = {
@@ -43,7 +50,8 @@ def invoke(self, request):
                 headers=headers
             ).json()
         else:
-            response = self.client.chat.completions.create(**request)
+            # response = self.client.chat.completions.create(**request)
+            response = self.openai_create_helper(self.client.chat.completions.create,request)
         logger.info(f"response:{response},{request}")
 
         if request.get("stream", False):
@@ -58,7 +66,7 @@ async def ainvoke(self, request):
         logger.info(f"Chat request:{request}")
         if self.model_type == ModelType.EMBEDDING:
             # print('cal embedding....')
-            response = await self.async_client.embeddings.create(**request)
+            response = await self.openai_create_helper(self.async_client.embeddings.create,request)
             # print('end cal embedding....')
         elif self.model_type == ModelType.RERANK:
             headers = {
@@ -71,7 +79,10 @@ async def ainvoke(self, request):
                 headers=headers
             ).json()
         else:
-            response = await self.async_client.chat.completions.create(**request)
+            response = await self.openai_create_helper(
+                self.async_client.chat.completions.create,
+                request
+            )
         logger.info(f"response:{response},{request}")
 
         if request.get("stream", False):
