fix cell_ to module_ in mindnlp.peft (#1874)

Author: lvyufeng

mindnlp/peft/peft_model.py

@@ -474,7 +474,7 @@ def __init__(self, model, peft_config: PeftConfig, adapter_name="default"):
             self.modules_to_save.update({"classifier", "score"})
 
         for name, _ in self.base_model.cells_and_names():
-            if any(cell_name in name for cell_name in self.modules_to_save):
+            if any(module_name in name for module_name in self.modules_to_save):
                 self.cls_layer_name = name
                 break
 
@@ -956,7 +956,7 @@ def __init__(self, model, peft_config: PeftConfig = None, adapter_name="default"
             self.modules_to_save.update({"classifier", "score"})
 
         for name, _ in self.base_model.cells_and_names():
-            if any(cell_name in name for cell_name in self.modules_to_save):
+            if any(module_name in name for module_name in self.modules_to_save):
                 self.cls_layer_name = name
                 break
 

mindnlp/peft/tuners/ia3/model.py

@@ -37,7 +37,7 @@
     ModulesToSaveWrapper,
     _get_subcells,
 )
-from ..tuners_utils import BaseTuner, BaseTunerLayer, check_target_cell_exists
+from ..tuners_utils import BaseTuner, BaseTunerLayer, check_target_module_exists
 from .layer import Conv2d, IA3Layer, Linear
 
 
@@ -192,7 +192,7 @@ def _create_new_cell(ia3_config, adapter_name, target, **kwargs):
         return new_cell
 
     @staticmethod
-    def _check_target_cell_exists(ia3_config, key):
+    def _check_target_module_exists(ia3_config, key):
         r"""
         Checks if the target cell exists in the IA3 configuration.
         
@@ -209,7 +209,7 @@ def _check_target_cell_exists(ia3_config, key):
         Raises:
             None: This method does not raise any exceptions.
         """
-        return check_target_cell_exists(ia3_config, key)
+        return check_target_module_exists(ia3_config, key)
 
     def _mark_only_adapters_as_trainable(self, model: nn.Module) -> None:
         r"""
@@ -274,7 +274,7 @@ def _create_and_replace(self, ia3_config, adapter_name, target, target_name, par
                 None
             """
             current_key = optionnal_kwargs.pop('current_key')
-            is_feedforward = self._check_target_cell_feedforward(ia3_config, current_key)
+            is_feedforward = self._check_target_module_feedforward(ia3_config, current_key)
             kwargs = {'fan_in_fan_out': ia3_config.fan_in_fan_out, 'init_ia3_weights': ia3_config.init_ia3_weights, 'is_feedforward': is_feedforward}
             kwargs['loaded_in_8bit'] = optionnal_kwargs.pop('loaded_in_8bit', False)
             kwargs['loaded_in_4bit'] = optionnal_kwargs.pop('loaded_in_4bit', False)
@@ -287,7 +287,7 @@ def _create_and_replace(self, ia3_config, adapter_name, target, target_name, par
                 self._replace_cell(parent, target_name, new_cell, target)
         # check if target cell is in feedforward_cells
         current_key = optionnal_kwargs.pop("current_key")
-        is_feedforward = self._check_target_cell_feedforward(ia3_config, current_key)
+        is_feedforward = self._check_target_module_feedforward(ia3_config, current_key)
 
         kwargs = {
             "fan_in_fan_out": ia3_config.fan_in_fan_out,
@@ -309,7 +309,7 @@ def _create_and_replace(self, ia3_config, adapter_name, target, target_name, par
             self._replace_cell(parent, target_name, new_cell, target)
 
     @staticmethod
-    def _check_target_cell_feedforward(ia3_config, key) -> bool:
+    def _check_target_module_feedforward(ia3_config, key) -> bool:
         """
         A helper private method that checks if the target cell `key` matches with a feedforward cell specified in
         `ia3_config`

mindnlp/peft/tuners/ln_tuning/model.py

@@ -26,7 +26,7 @@
 from mindnlp.peft.tuners.tuners_utils import (
     BaseTuner,
     _get_subcells,
-    check_target_cell_exists,
+    check_target_module_exists,
 )
 from mindnlp.peft.utils import (
     TRANSFORMERS_MODELS_TO_LNTUNING_TARGET_MODULES_MAPPING,
@@ -150,8 +150,8 @@ def _mark_only_adapters_as_trainable(self, model: Cell):
             else:
                 p.requires_grad = True
 
-    def _check_target_cell_exists(self, peft_config: PeftConfig, key: str) -> bool:
-        return check_target_cell_exists(peft_config, key)
+    def _check_target_module_exists(self, peft_config: PeftConfig, key: str) -> bool:
+        return check_target_module_exists(peft_config, key)
 
     def _set_adapter_layers(self, enabled: bool) -> None:
         for cell in self.model.cells():

mindnlp/peft/tuners/lokr/config.py

@@ -30,7 +30,7 @@ class LoKrConfig(PeftConfig):
         target_modules (`Union[List[str],str]`): The names of the cells to apply Lora to.
         lora_alpha (`float`): The alpha parameter for Lokr scaling.
         rank_dropout (`float`):The dropout probability for rank dimension during training.
-        cell_dropout (`float`): The dropout probability for LoKR layers.
+        module_dropout (`float`): The dropout probability for LoKR layers.
         use_effective_conv2d (`bool`):
             Use parameter effective decomposition for
             Conv2d with ksize > 1 ("Proposition 3" from FedPara paper).
@@ -71,7 +71,7 @@ class LoKrConfig(PeftConfig):
         default=0.0,
         metadata={"help": "The dropout probability for rank dimension during training"},
     )
-    cell_dropout: float = field(default=0.0, metadata={"help": "lokr dropout"})
+    module_dropout: float = field(default=0.0, metadata={"help": "lokr dropout"})
     use_effective_conv2d: bool = field(
         default=False,
         metadata={

mindnlp/peft/tuners/lokr/layer.py

@@ -49,7 +49,7 @@ class LoKrLayer(nn.Module, BaseTunerLayer):
         - alpha: Dictionary storing alpha values for each adapter.
         - scaling: Dictionary storing scaling values for each adapter.
         - rank_dropout: Dictionary storing rank dropout probabilities for each adapter.
-        - cell_dropout: Dictionary storing cell dropout probabilities for each adapter.
+        - module_dropout: Dictionary storing cell dropout probabilities for each adapter.
         - _disable_adapters: Boolean flag indicating whether adapters are disabled.
         - merged_adapters: List of names of merged adapters.
     
@@ -74,7 +74,7 @@ class LoKrLayer(nn.Module, BaseTunerLayer):
     Note:
         This class is intended for advanced neural network adaptation techniques and should be used in conjunction with PyTorch's nn.Module functionalities.
     """
-    other_param_names = ("r", "alpha", "scaling", "rank_dropout", "cell_dropout")
+    other_param_names = ("r", "alpha", "scaling", "rank_dropout", "module_dropout")
     # All names of layers that may contain adapter weights
     adapter_layer_names = (
         "lokr_w1",
@@ -132,7 +132,7 @@ def __init__(self, base_layer: nn.Module) -> None:
         self.alpha = {}
         self.scaling = {}
         self.rank_dropout = {}
-        self.cell_dropout = {}
+        self.module_dropout = {}
 
         # Tuner info
         self._disable_adapters = False
@@ -491,7 +491,7 @@ def update_layer(
         r: int,
         alpha: float,
         rank_dropout: float,
-        cell_dropout: float,
+        module_dropout: float,
         init_weights: bool,
         use_effective_conv2d: bool,
         decompose_both: bool,
@@ -505,7 +505,7 @@ def update_layer(
             r (`int`): Rank for the added adapter.
             alpha (`float`): Alpha for the added adapter.
             rank_dropout (`float`): The dropout probability for rank dimension during training
-            cell_dropout (`float`): The dropout probability for disabling adapter during training.
+            module_dropout (`float`): The dropout probability for disabling adapter during training.
             init_weights (`bool`): Whether to initialize adapter weights.
             use_effective_conv2d (`bool`): Use parameter effective decomposition for Conv2d with ksize > 1.
             decompose_both (`bool`): Perform rank decomposition of left kronecker product matrix.
@@ -520,7 +520,7 @@ def update_layer(
         self.alpha[adapter_name] = alpha
         self.scaling[adapter_name] = alpha / r
         self.rank_dropout[adapter_name] = rank_dropout
-        self.cell_dropout[adapter_name] = cell_dropout
+        self.module_dropout[adapter_name] = module_dropout
         base_layer = self.get_base_layer()
 
         # Determine shape of LoKr weights
@@ -589,8 +589,8 @@ def set_adapter(self, adapter_names) -> None:
 
         # Deactivate grads on the inactive adapter and activate grads on the active adapter
         for layer_name in self.adapter_layer_names:
-            cell_dict = getattr(self, layer_name)
-            for key, layer in cell_dict.items():
+            module_dict = getattr(self, layer_name)
+            for key, layer in module_dict.items():
                 if key in adapter_names:
                     # Note: It is possible that not a single layer is called with requires_grad_(True) here. This may
                     # happen if a completely different adapter layer is being activated.
@@ -729,11 +729,11 @@ def forward(self, x: ms.Tensor, *args, **kwargs) -> ms.Tensor:
                 if active_adapter not in self._available_adapters:
                     continue
 
-                cell_dropout = self.cell_dropout[active_adapter]
+                module_dropout = self.module_dropout[active_adapter]
 
                 # Modify current execution weights
                 if (not self.training) or (
-                    self.training and ops.rand(1) > cell_dropout
+                    self.training and ops.rand(1) > module_dropout
                 ):
                     result = result + self._get_delta_activations(
                         active_adapter, x, *args, **kwargs
@@ -752,7 +752,7 @@ def __init__(
         r: int = 0,
         alpha: float = 0.0,
         rank_dropout: float = 0.0,
-        cell_dropout: float = 0.0,
+        module_dropout: float = 0.0,
         init_weights: bool = True,
         **kwargs,
     ):
@@ -766,7 +766,7 @@ def __init__(
             r (int): The value of r for adapter update. Defaults to 0.
             alpha (float): The value of alpha for adapter update. Defaults to 0.0.
             rank_dropout (float): The dropout value for rank. Defaults to 0.0.
-            cell_dropout (float): The dropout value for cell. Defaults to 0.0.
+            module_dropout (float): The dropout value for cell. Defaults to 0.0.
             init_weights (bool): A flag to initialize weights. Defaults to True.
             **kwargs: Additional keyword arguments.
         
@@ -781,7 +781,7 @@ def __init__(
         # Create adapter and set it active
         self._active_adapter = adapter_name
         self.update_layer(
-            adapter_name, r, alpha, rank_dropout, cell_dropout, init_weights, **kwargs
+            adapter_name, r, alpha, rank_dropout, module_dropout, init_weights, **kwargs
         )
 
     def _get_delta_activations(
@@ -835,7 +835,7 @@ def __init__(
         r: int = 0,
         alpha: float = 0.0,
         rank_dropout: float = 0.0,
-        cell_dropout: float = 0.0,
+        module_dropout: float = 0.0,
         use_effective_conv2d: bool = False,
         init_weights: bool = True,
         **kwargs,
@@ -850,7 +850,7 @@ def __init__(
             r (int): The value of parameter 'r'.
             alpha (float): The value of parameter 'alpha'.
             rank_dropout (float): The value of rank dropout.
-            cell_dropout (float): The value of cell dropout.
+            module_dropout (float): The value of cell dropout.
             use_effective_conv2d (bool): Flag indicating whether to use effective Conv2d.
             init_weights (bool): Flag indicating whether to initialize weights.
             **kwargs: Additional keyword arguments.
@@ -870,7 +870,7 @@ def __init__(
             r,
             alpha,
             rank_dropout,
-            cell_dropout,
+            module_dropout,
             init_weights,
             use_effective_conv2d,
             **kwargs,

mindnlp/peft/tuners/lokr/model.py

@@ -28,7 +28,7 @@
 from ..tuners_utils import (
     BaseTuner,
     BaseTunerLayer,
-    check_target_cell_exists,
+    check_target_module_exists,
 )
 from .layer import Conv2d, Dense, LoKrLayer
 from .config import LoKrConfig
@@ -59,7 +59,7 @@ class LoKrModel(BaseTuner):
         ...     lora_alpha=32,
         ...     target_modules=["k_proj", "q_proj", "v_proj", "out_proj", "fc1", "fc2"],
         ...     rank_dropout=0.0,
-        ...     cell_dropout=0.0,
+        ...     module_dropout=0.0,
         ...     init_weights=True,
         ... )
         >>> config_unet = LoKrConfig(
@@ -76,7 +76,7 @@ class LoKrModel(BaseTuner):
         ...         "ff.net.2",
         ...     ],
         ...     rank_dropout=0.0,
-        ...     cell_dropout=0.0,
+        ...     module_dropout=0.0,
         ...     init_weights=True,
         ...     use_effective_conv2d=True,
         ... )
@@ -153,22 +153,22 @@ def _create_new_cell(
                 This occurs when the target cell type does not match any of the supported cell types in the layers_mapping attribute.
         """
         # Find corresponding subtype of provided target cell
-        new_cell_cls = None
+        new_module_cls = None
         for subtype, target_cls in cls.layers_mapping.items():
             if (
                 hasattr(target, "base_layer")
                 and isinstance(target.get_base_layer(), subtype)
                 and isinstance(target, BaseTunerLayer)
             ):
                 # nested tuner layers are allowed
-                new_cell_cls = target_cls
+                new_module_cls = target_cls
                 break
             elif isinstance(target, subtype):
-                new_cell_cls = target_cls
+                new_module_cls = target_cls
                 break
 
         # We didn't find corresponding type, so adapter for this layer is not supported
-        if new_cell_cls is None:
+        if new_module_cls is None:
             supported_cells = ", ".join(
                 layer.__name__ for layer in cls.layers_mapping.keys()
             )
@@ -183,9 +183,9 @@ def _create_new_cell(
             target_base_layer = target
 
         if isinstance(target_base_layer, nn.Module):
-            new_cell = new_cell_cls(target, adapter_name=adapter_name, **kwargs)
+            new_cell = new_module_cls(target, adapter_name=adapter_name, **kwargs)
         elif isinstance(target_base_layer, nn.Module):
-            new_cell = new_cell_cls(target, adapter_name=adapter_name, **kwargs)
+            new_cell = new_module_cls(target, adapter_name=adapter_name, **kwargs)
         else:
             supported_cells = ", ".join(
                 layer.__name__ for layer in cls.layers_mapping.keys()
@@ -388,7 +388,7 @@ def _prepare_adapter_config(peft_config, model_config):
         return peft_config
 
     @staticmethod
-    def _check_target_cell_exists(LoKR_config, key):
+    def _check_target_module_exists(LoKR_config, key):
         r"""
         Checks if a target cell exists in the LoKR configuration.
         
@@ -403,4 +403,4 @@ def _check_target_cell_exists(LoKR_config, key):
             This method does not raise any exceptions explicitly. However, if the target cell does not exist in the LoKR configuration, further handling may be required based on the context in which this
 method is used.
         """
-        return check_target_cell_exists(LoKR_config, key)
+        return check_target_module_exists(LoKR_config, key)

mindnlp/peft/tuners/lora/model.py

@@ -36,7 +36,7 @@
 from ..tuners_utils import (
     BaseTuner,
     BaseTunerLayer,
-    check_target_cell_exists,
+    check_target_module_exists,
     # onload_layer,
     replicate_layers,
 )
@@ -161,7 +161,7 @@ def _check_new_adapter_config(self, config: LoraConfig) -> None:
             )
 
     @staticmethod
-    def _check_target_cell_exists(lora_config, key):
+    def _check_target_module_exists(lora_config, key):
         r"""
         Checks if the target cell exists in the LoRa configuration.
         
@@ -190,7 +190,7 @@ def _check_target_cell_exists(lora_config, key):
         Raises:
             None: This method does not raise any exceptions.
         """
-        return check_target_cell_exists(lora_config, key)
+        return check_target_module_exists(lora_config, key)
 
     def _prepare_model(self, peft_config: LoraConfig, model: nn.Module):
         r"""
@@ -683,23 +683,23 @@ def _check_add_weighted_adapter(
         else:
             raise ValueError(f"Invalid combination_type: {combination_type}")
 
-        target_cell_types = [type(self.peft_config[adapter].target_modules) for adapter in adapters]
-        if not target_cell_types:
+        target_module_types = [type(self.peft_config[adapter].target_modules) for adapter in adapters]
+        if not target_module_types:
             raise ValueError(f"Found no adapter matching the names in {adapters}")
-        if len(set(target_cell_types)) > 1:
+        if len(set(target_module_types)) > 1:
             raise ValueError(
                 "all adapter configs should follow the same target cells type. "
                 "Combining adapters with `target_modules` type being a mix of list/set and string is not supported."
             )
 
-        if target_cell_types[0] == str:
+        if target_module_types[0] == str:
             new_target_modules = "|".join(f"({self.peft_config[adapter].target_modules})" for adapter in adapters)
-        elif target_cell_types[0] == set:
+        elif target_module_types[0] == set:
             new_target_modules = reduce(
                 operator.or_, (self.peft_config[adapter].target_modules for adapter in adapters)
             )
         else:
-            raise TypeError(f"Invalid type {target_cell_types[0]} found in target_modules")
+            raise TypeError(f"Invalid type {target_module_types[0]} found in target_modules")
 
         return combination_type, new_rank, new_target_modules