feat(transformers): suplement glm4v processor

Author: wcrzlh

examples/transformers/glm4v/generate.py

@@ -1,11 +1,10 @@
 import argparse
 
 import numpy as np
-from transformers import AutoProcessor
 
 import mindspore as ms
 
-from mindone.transformers import Glm4vForConditionalGeneration
+from mindone.transformers import AutoProcessor, Glm4vForConditionalGeneration
 
 
 def generate(args):

mindone/transformers/__init__.py

@@ -1531,8 +1531,10 @@
 if version.parse(transformers.__version__) >= version.parse("4.53.0"):
     from .models.glm4v import (
         Glm4vForConditionalGeneration,
+        Glm4vImageProcessor,
         Glm4vModel,
         Glm4vPreTrainedModel,
+        Glm4vProcessor,
         Glm4vTextModel,
         Glm4vVisionModel,
     )

mindone/transformers/integrations/flash_attention.py

@@ -69,6 +69,9 @@ def flash_attention_forward(
     if kwargs.get("is_causal", None) is not None:
         kwargs.pop("is_causal")
 
+    if not hasattr(module, "is_causal"):
+        module.is_causal = False
+
     attn_output = _flash_attention_forward(
         query,
         key,

mindone/transformers/models/auto/image_processing_auto.py

@@ -62,19 +62,20 @@
             ("dpt", ("DPTImageProcessor",)),
             ("efficientnet", ("EfficientNetImageProcessor",)),
             ("flava", ("FlavaImageProcessor",)),
-            ("oneformer", ("OneFormerImageProcessor",)),
+            ("glm4v", ("Glm4vImageProcessor",)),
             ("llava_next", ("LlavaNextImageProcessor",)),
             ("llava_next_video", ("LlavaNextVideoImageProcessor",)),
             ("llava_onevision", ("LlavaOnevisionImageProcessor",)),
             ("maskformer", ("MaskFormerImageProcessor",)),
             ("mllama", ("MllamaImageProcessor",)),
-            ("qwen2_5_vl", ("Qwen2VLImageProcessor",)),
+            ("oneformer", ("OneFormerImageProcessor",)),
             ("owlv2", ("Owlv2ImageProcessor",)),
             ("owlvit", ("OwlViTImageProcessor",)),
-            ("videomae", ("VideoMAEImageProcessor",)),
+            ("qwen2_5_vl", ("Qwen2VLImageProcessor",)),
             ("sam", ("SamImageProcessor",)),
             ("segformer", ("SegformerImageProcessor",)),
             ("siglip", ("SiglipImageProcessor", "SiglipImageProcessorFast")),
+            ("videomae", ("VideoMAEImageProcessor",)),
             ("yolos", ("YolosImageProcessor",)),
         ]
     )

mindone/transformers/models/auto/processing_auto.py

@@ -54,18 +54,19 @@
         ("chinese_clip", "ChineseCLIPProcessor"),
         ("colpali", "ColPaliProcessor"),
         ("flava", "FlavaProcessor"),
+        ("glm4v", "Glm4vProcessor"),
         ("idefics", "IdeficsProcessor"),
-        ("layoutlmv3", "LayoutMv3Processor"),
         ("instructblip", "InstructBlipProcessor"),
+        ("layoutlmv3", "LayoutMv3Processor"),
         ("llava_next", "LlavaNextProcessor"),
         ("llava_next_video", "LlavaNextVideoProcessor"),
         ("llava_onevision", "LlavaOnevisionProcessor"),
-        ("pop2piano", "Pop2PianoProcessor"),
-        ("qwen2_5_vl", "Qwen2_5_VLProcessor"),
+        ("oneformer", "OneFormerProcessor"),
         ("owlv2", "Owlv2Processor"),
         ("owlvit", "OwlViTProcessor"),
+        ("pop2piano", "Pop2PianoProcessor"),
+        ("qwen2_5_vl", "Qwen2_5_VLProcessor"),
         ("sam", "SamProcessor"),
-        ("oneformer", "OneFormerProcessor"),
         ("seamless_m4t", "SeamlessM4TProcessor"),
         ("siglip", "SiglipProcessor"),
     ]

mindone/transformers/models/glm4v/__init__.py

@@ -15,10 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from .image_processing_glm4v import Glm4vImageProcessor
 from .modeling_glm4v import (
     Glm4vForConditionalGeneration,
     Glm4vModel,
     Glm4vPreTrainedModel,
     Glm4vTextModel,
     Glm4vVisionModel,
 )
+from .processing_glm4v import Glm4vProcessor