feat(transformers): supplement glm4v processor (#1349)

* feat(transformers): suplement glm4v processor

* update ms 2.7.0 perf

Author: wcrzlh

examples/transformers/glm4v/README.md

@@ -15,9 +15,9 @@ Provides an open-source version supporting both Chinese and English bilingual us
 # Get Started
 
 ## Requirements:
-| mindspore   | 	ascend driver | firmware       | cann tookit/kernel|
-|-------------|----------------|----------------| --- |
-| 2.5.0/2.6.0 | 24.1.RC3.b080  | 7.5.T11.0.B088 | 8.0.RC3.beta1|
+| mindspore   | 	ascend driver | firmware       | cann tookit/kernel |
+|-------------|----------------|----------------|--------------------|
+| 2.6.0/2.7.0 | 24.1.RC3.b080  | 7.5.T11.0.B088 | 8.1.RC1            |
 
 ### Installation:
 ```
@@ -45,5 +45,5 @@ python generate.py
 ## Inference Speed
 |      model name	      | mindspore version |   precision*   | cards | flash attn | 	tokens/s	 |
 |:---------------------:|:-----------------:|:--------------:|:---:  |:----------:|:----------:|
-| THUDM/GLM-4.1V-9B-Thinking |       2.5.0       |   bf16 | 1 |     ✅      |    1.38    |
 | THUDM/GLM-4.1V-9B-Thinking |       2.6.0       |   bf16 | 1 |     ✅      |    1.63    |
+| THUDM/GLM-4.1V-9B-Thinking |       2.7.0       |   bf16 | 1 |     ✅      |    1.66     |

examples/transformers/glm4v/generate.py

@@ -1,11 +1,10 @@
 import argparse
 
 import numpy as np
-from transformers import AutoProcessor
 
 import mindspore as ms
 
-from mindone.transformers import Glm4vForConditionalGeneration
+from mindone.transformers import AutoProcessor, Glm4vForConditionalGeneration
 
 
 def generate(args):

mindone/transformers/__init__.py

@@ -1531,8 +1531,10 @@
 if version.parse(transformers.__version__) >= version.parse("4.53.0"):
     from .models.glm4v import (
         Glm4vForConditionalGeneration,
+        Glm4vImageProcessor,
         Glm4vModel,
         Glm4vPreTrainedModel,
+        Glm4vProcessor,
         Glm4vTextModel,
         Glm4vVisionModel,
     )

mindone/transformers/integrations/flash_attention.py

@@ -69,6 +69,9 @@ def flash_attention_forward(
     if kwargs.get("is_causal", None) is not None:
         kwargs.pop("is_causal")
 
+    if not hasattr(module, "is_causal"):
+        module.is_causal = False
+
     attn_output = _flash_attention_forward(
         query,
         key,

mindone/transformers/models/auto/image_processing_auto.py

@@ -27,6 +27,8 @@
 from typing import TYPE_CHECKING, Dict, Optional, Tuple, Union
 
 # Build the list of all image processors
+import transformers
+from packaging import version
 from transformers import PretrainedConfig
 from transformers.dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
 from transformers.utils import CONFIG_NAME, cached_file
@@ -79,6 +81,10 @@
         ]
     )
 
+
+if version.parse(transformers.__version__) >= version.parse("4.53.0"):
+    IMAGE_PROCESSOR_MAPPING_NAMES.update({"glm4v": ("Glm4vImageProcessor",)})
+
 for model_type, image_processors in IMAGE_PROCESSOR_MAPPING_NAMES.items():
     slow_image_processor_class, *fast_image_processor_class = image_processors
     if not is_vision_available():

mindone/transformers/models/auto/processing_auto.py

@@ -25,6 +25,8 @@
 from collections import OrderedDict
 
 # Build the list of all feature extractors
+import transformers
+from packaging import version
 from transformers.configuration_utils import PretrainedConfig
 from transformers.dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
 from transformers.models.auto.tokenization_auto import AutoTokenizer
@@ -71,6 +73,9 @@
     ]
 )
 
+if version.parse(transformers.__version__) >= version.parse("4.53.0"):
+    PROCESSOR_MAPPING_NAMES.update({"glm4v": "Glm4vProcessor"})
+
 PROCESSOR_MAPPING = _LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)
 
 

mindone/transformers/models/glm4v/__init__.py

@@ -15,10 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .image_processing_glm4v import Glm4vImageProcessor
 from .modeling_glm4v import (
     Glm4vForConditionalGeneration,
     Glm4vModel,
     Glm4vPreTrainedModel,
     Glm4vTextModel,
     Glm4vVisionModel,
 )
+from .processing_glm4v import Glm4vProcessor