FIX DeLoRA adapter deletion issue

Currently, adapter deletion raises an error with DeLoRA. The reason is
that the dropout module is called module_dropout, i.e. the prefix
"delora" is not part of the name, which is required for proper working.
This PR renames the attribute to delora_dropout.

The reason why this was not caught is because the corresponding test was
not updated to include DeLoRA. The test was thus also changed.

Note: I came across this issue in huggingface#2846 but I wanted to fix it in a
separate PR, as huggingface#2846 is probably not going to make it into the next
PEFT release.

Author: BenjaminBossan

src/peft/tuners/delora/layer.py

@@ -34,14 +34,14 @@ class DeloraLayer(BaseTunerLayer):
     # All names of other parameters that may contain adapter-related parameters
     other_param_names = (
         "r",
-        "module_dropout",
+        "delora_dropout",
         "delora_w_norm",
     )
 
     def __init__(self, base_layer: nn.Module, **kwargs) -> None:
         self.base_layer = base_layer
         self.r = {}
-        self.module_dropout = nn.ModuleDict({})
+        self.delora_dropout = nn.ModuleDict({})
         self.delora_A = nn.ParameterDict({})
         self.delora_B = nn.ParameterDict({})
         self.delora_lambda = nn.ParameterDict({})
@@ -113,7 +113,7 @@ def update_layer(
             module_dropout_layer = nn.Dropout(p=module_dropout)
         else:
             module_dropout_layer = nn.Identity()
-        self.module_dropout.update(nn.ModuleDict({adapter_name: module_dropout_layer}))
+        self.delora_dropout.update(nn.ModuleDict({adapter_name: module_dropout_layer}))
 
         # Initialize weights
         self.reset_delora_parameters(adapter_name, init_weights, delora_lambda)
@@ -241,7 +241,7 @@ def forward(self, x: torch.Tensor, *args: Any, **kwargs: Any) -> torch.Tensor:
                 if adapter not in self.delora_A:
                     continue
 
-                x_d = self.module_dropout[adapter](x)
+                x_d = self.delora_dropout[adapter](x)
 
                 # Decomposed delta calculation
                 # 1. (x * w_norm) @ A.T

tests/testing_common.py

@@ -1434,6 +1434,7 @@ def _test_delete_adapter(self, model_id, config_cls, config_kwargs):
             PeftType.VBLORA,
             PeftType.BONE,
             PeftType.MISS,
+            PeftType.DELORA,
         ]
         # IA3 does not support deleting adapters yet, but it just needs to be added
         # AdaLora does not support multiple adapters