support fill_mask/image_classification/image_feature_extraction pipelines (#1801)

Author: lvyufeng

mindnlp/core/ops/other.py

@@ -620,7 +620,9 @@ def repeat_interleave(input, repeats, dim=None):
 
 # roll
 def roll(input, shifts, dims=None):
-    return mindspore.numpy.roll(input, shifts, dims)
+    if use_pyboost():
+        return mindspore.mint.roll(input, shifts, dims)
+    return ops.roll(input, shifts, dims)
 
 # searchsorted
 def searchsorted(sorted_sequence, values, *, out_int32=False, right=False, side=None, sorter=None):

mindnlp/mimm/__init__.py

@@ -0,0 +1 @@
+"""mindspore image models"""

mindnlp/mimm/models/__init__.py

@@ -138,6 +138,7 @@
     marian,
     markuplm,
     m2m_100,
+    maskformer,
     mask2former,
     mbart,
     mbart50,
@@ -380,6 +381,7 @@
 from .mamba import *
 from .marian import *
 from .markuplm import *
+from .maskformer import *
 from .mask2former import *
 from .mbart import *
 from .mbart50 import *
@@ -622,6 +624,7 @@
 __all__.extend(mamba.__all__)
 __all__.extend(marian.__all__)
 __all__.extend(markuplm.__all__)
+__all__.extend(maskformer.__all__)
 __all__.extend(mask2former.__all__)
 __all__.extend(mbart.__all__)
 __all__.extend(mbart50.__all__)

mindnlp/transformers/models/__init__.py

@@ -70,6 +70,8 @@
     MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING,
     MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING,
     MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING,
+    MODEL_FOR_IMAGE_SEGMENTATION_MAPPING,
+    MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING,
     MODEL_MAPPING,
     MODEL_WITH_LM_HEAD_MAPPING,
     AutoBackbone,
@@ -105,6 +107,7 @@
     AutoModelForZeroShotImageClassification,
     AutoModelForZeroShotObjectDetection,
     AutoModelWithLMHead,
+    AutoModelForImageSegmentation,
 )
 
 __all__ = [
@@ -127,6 +130,7 @@
     'MODEL_FOR_DEPTH_ESTIMATION_MAPPING',
     'MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING',
     'MODEL_FOR_IMAGE_TO_IMAGE_MAPPING',
+    'MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING',
     'MODEL_FOR_INSTANCE_SEGMENTATION_MAPPING',
     'MODEL_FOR_MASK_GENERATION_MAPPING',
     'MODEL_FOR_MASKED_LM_MAPPING',
@@ -152,6 +156,8 @@
     'MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING',
     'MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING',
     'MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING',
+    'MODEL_FOR_IMAGE_SEGMENTATION_MAPPING',
+    'MODEL_FOR_SEMANTIC_SEGMENTATION_MAPPING',
     'MODEL_MAPPING',
     'MODEL_WITH_LM_HEAD_MAPPING',
     'AutoBackbone',
@@ -187,4 +193,5 @@
     'AutoModelForZeroShotImageClassification',
     'AutoModelForZeroShotObjectDetection',
     'AutoModelWithLMHead',
+    'AutoModelForImageSegmentation'
 ]

mindnlp/transformers/models/auto/__init__.py

@@ -138,6 +138,8 @@
         ("marian", "MarianConfig"),
         ('markuplm', "MarkupLMConfig"),
         ("mask2former", "Mask2FormerConfig"),
+        ("maskformer", "MaskFormerConfig"),
+        ("maskformer-swin", "MaskFormerSwinConfig"),
         ("mbart", "MBartConfig"),
         ("mctct", "MCTCTConfig"),
         ("megatron-bert", 'MegatronBertConfig'),

mindnlp/transformers/models/auto/configuration_auto.py

@@ -679,7 +679,7 @@
         ("mobilevit", "MobileViTForSemanticSegmentation"),
         ("mobilevitv2", "MobileViTV2ForSemanticSegmentation"),
         ("segformer", "SegformerForSemanticSegmentation"),
-        ("upernet", "UperNetForSemanticSegmentation"),
+        # ("upernet", "UperNetForSemanticSegmentation"),
     ]
 )
 

mindnlp/transformers/models/auto/modeling_auto.py

@@ -15,7 +15,7 @@
 # ============================================================================
 """ BEiT model configuration"""
 from mindnlp.utils import logging
-from ...backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices
+from ....utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices
 from ...configuration_utils import PretrainedConfig
 
 

mindnlp/transformers/models/beit/configuration_beit.py

@@ -30,7 +30,7 @@
 from mindnlp.core.nn import Parameter
 from mindnlp.core.nn import functional as F
 from mindnlp.utils import logging
-from ...backbone_utils import BackboneMixin
+from ....utils.backbone_utils import BackboneMixin
 from ...activations import ACT2FN
 from ...modeling_outputs import (
     BackboneOutput,

mindnlp/transformers/models/beit/modeling_beit.py

@@ -16,7 +16,7 @@
 
 from ...configuration_utils import PretrainedConfig
 from ....utils import logging
-from ...backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices
+from ....utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices
 
 
 logger = logging.get_logger(__name__)