From 95be3cf7059073f70c2da16f16d117d90cf7e2a9 Mon Sep 17 00:00:00 2001 From: calpt Date: Sun, 22 Dec 2024 13:02:27 +0100 Subject: [PATCH 1/9] Fix default Lora/ (IA)^3 scaling in forward --- src/adapters/methods/lora.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/adapters/methods/lora.py b/src/adapters/methods/lora.py index c62a94f265..0a010cc0b4 100644 --- a/src/adapters/methods/lora.py +++ b/src/adapters/methods/lora.py @@ -99,6 +99,7 @@ def forward(self, hidden_states: Optional[torch.Tensor], layer_input: torch.Tens hidden_states = hidden_states * gate else: gate = None + hidden_states = hidden_states * self.scaling return hidden_states, gate @@ -170,6 +171,7 @@ def forward(self, hidden_states: Optional[torch.Tensor], layer_input: torch.Tens hidden_states = hidden_states * gate else: gate = None + hidden_states = hidden_states * self.scaling return hidden_states, gate From 58522c6abe99e18ec7aa5645f4f6e3fb4a4e23a5 Mon Sep 17 00:00:00 2001 From: calpt Date: Mon, 23 Dec 2024 12:10:49 +0100 Subject: [PATCH 2/9] Add backward compat logic. --- setup.py | 4 +++- src/adapters/__init__.py | 2 +- src/adapters/loading.py | 13 +++++++++++++ 3 files changed, 17 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 6fd87f9ca9..fb464e07cf 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ "isort>=5.5.4", "Jinja2==2.11.3", "nltk", + "packaging", "parameterized", "pillow", "protobuf", @@ -136,11 +137,12 @@ def deps_list(*pkgs): # when modifying the following list, make sure to update src/transformers/dependency_versions_check.py install_requires = [ deps["transformers"], + deps["packaging"], ] setup( name="adapters", - version="1.0.1", + version="1.1.0.dev0", author="The AdapterHub team and community contributors", author_email="calpt@mail.de", description="A Unified Library for Parameter-Efficient and Modular Transfer Learning", diff --git a/src/adapters/__init__.py b/src/adapters/__init__.py index a917828e72..88549c6969 100644 --- a/src/adapters/__init__.py +++ b/src/adapters/__init__.py @@ -16,7 +16,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -__version__ = "1.0.1" +__version__ = "1.1.0.dev0" from typing import TYPE_CHECKING diff --git a/src/adapters/loading.py b/src/adapters/loading.py index b1918b0a0f..a43b3d5257 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -6,6 +6,7 @@ from typing import Callable, Mapping, Optional, Sequence, Tuple import torch +from packaging.version import Version try: @@ -368,6 +369,16 @@ def _rename_legacy_weights(self, k): k = k.replace(old, new) return k + def _fix_backward_compat(self, config): + # Fix error in previous versions for LoRA/ (IA)^3 + if config["version"].startswith("adapters.") and Version(config["version"][9:]) < Version("1.1.0"): + if config["config"]["architecture"] == "lora" and config["config"]["r"] != config["config"]["alpha"]: + logger.warning( + "Loading a LoRA trained using a faulty library version. Editing the configuration to make sure the adapter works as trained." + "See https://github.com/adapter-hub/adapters/pull/770 for more." + ) + config["config"]["alpha"] = config["config"]["r"] + # This method is used to remove unnecessary invertible adapters from task adapters using the old format. # In the old format, task adapters e.g. using seq_bn config specify inv. adapters but don't use them. # As inv. adapters would be incorrectly used in the new implementation, @@ -560,6 +571,8 @@ def load( # The conversion to a set and then back to a list removes all duplicates leave_out = list(set(leave_out + config["config"]["leave_out"])) config["config"]["leave_out"] = leave_out + # Fix issues + self._fix_backward_compat(config) adapter_name = load_as or config["name"] # If the adapter is not part of the model, add it From 51f040f70b32373a7a3d8c1ba8a8879273faa2e7 Mon Sep 17 00:00:00 2001 From: calpt Date: Mon, 23 Dec 2024 12:24:37 +0100 Subject: [PATCH 3/9] fix --- src/adapters/loading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/loading.py b/src/adapters/loading.py index a43b3d5257..24e8b5a03f 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -372,7 +372,7 @@ def _rename_legacy_weights(self, k): def _fix_backward_compat(self, config): # Fix error in previous versions for LoRA/ (IA)^3 if config["version"].startswith("adapters.") and Version(config["version"][9:]) < Version("1.1.0"): - if config["config"]["architecture"] == "lora" and config["config"]["r"] != config["config"]["alpha"]: + if config["config"].get("architecture", None) == "lora" and config["config"]["r"] != config["config"]["alpha"]: logger.warning( "Loading a LoRA trained using a faulty library version. Editing the configuration to make sure the adapter works as trained." "See https://github.com/adapter-hub/adapters/pull/770 for more." From f8972bec9171dc9868478c10fc8b315837008005 Mon Sep 17 00:00:00 2001 From: calpt Date: Mon, 23 Dec 2024 12:37:27 +0100 Subject: [PATCH 4/9] style --- src/adapters/loading.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/adapters/loading.py b/src/adapters/loading.py index 24e8b5a03f..529d823d17 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -372,7 +372,10 @@ def _rename_legacy_weights(self, k): def _fix_backward_compat(self, config): # Fix error in previous versions for LoRA/ (IA)^3 if config["version"].startswith("adapters.") and Version(config["version"][9:]) < Version("1.1.0"): - if config["config"].get("architecture", None) == "lora" and config["config"]["r"] != config["config"]["alpha"]: + if ( + config["config"].get("architecture", None) == "lora" + and config["config"]["r"] != config["config"]["alpha"] + ): logger.warning( "Loading a LoRA trained using a faulty library version. Editing the configuration to make sure the adapter works as trained." "See https://github.com/adapter-hub/adapters/pull/770 for more." From 53f28004c6d95d9547cb046dbce3c3e3253d11b9 Mon Sep 17 00:00:00 2001 From: calpt Date: Mon, 23 Dec 2024 13:11:11 +0100 Subject: [PATCH 5/9] another fix --- src/adapters/loading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/loading.py b/src/adapters/loading.py index 529d823d17..44fd556c86 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -371,7 +371,7 @@ def _rename_legacy_weights(self, k): def _fix_backward_compat(self, config): # Fix error in previous versions for LoRA/ (IA)^3 - if config["version"].startswith("adapters.") and Version(config["version"][9:]) < Version("1.1.0"): + if config.get("version", "").startswith("adapters.") and Version(config["version"][9:]) < Version("1.1.0"): if ( config["config"].get("architecture", None) == "lora" and config["config"]["r"] != config["config"]["alpha"] From 086d7f2f8f9715195f7b318fce7f3e7a40f9045a Mon Sep 17 00:00:00 2001 From: calpt Date: Sun, 29 Dec 2024 16:14:56 +0100 Subject: [PATCH 6/9] Update src/adapters/loading.py Co-authored-by: TimoImhof <62378375+TimoImhof@users.noreply.github.com> --- src/adapters/loading.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/adapters/loading.py b/src/adapters/loading.py index 44fd556c86..d536842671 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -371,7 +371,11 @@ def _rename_legacy_weights(self, k): def _fix_backward_compat(self, config): # Fix error in previous versions for LoRA/ (IA)^3 - if config.get("version", "").startswith("adapters.") and Version(config["version"][9:]) < Version("1.1.0"): + ADAPTER_PREFIX = "adapters." + MIN_VERSION = Version("1.1.0") + + version = config.get("version", "") + if version.startswith(ADAPTER_PREFIX) and Version(version[len(ADAPTER_PREFIX):]) < MIN_VERSION: if ( config["config"].get("architecture", None) == "lora" and config["config"]["r"] != config["config"]["alpha"] From 8a2a90088459de30e3850130cfb26fa7b1823316 Mon Sep 17 00:00:00 2001 From: calpt Date: Sun, 29 Dec 2024 22:14:02 +0000 Subject: [PATCH 7/9] edit --- src/adapters/loading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/loading.py b/src/adapters/loading.py index d536842671..226ac5d74d 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -381,7 +381,7 @@ def _fix_backward_compat(self, config): and config["config"]["r"] != config["config"]["alpha"] ): logger.warning( - "Loading a LoRA trained using a faulty library version. Editing the configuration to make sure the adapter works as trained." + "Loading a LoRA trained using a faulty scaling implementation. Editing the configuration to make sure the adapter works as trained." "See https://github.com/adapter-hub/adapters/pull/770 for more." ) config["config"]["alpha"] = config["config"]["r"] From 3a6a759f6d728d4771c6c9ec3fbe7761aee32937 Mon Sep 17 00:00:00 2001 From: calpt Date: Sun, 29 Dec 2024 22:15:49 +0000 Subject: [PATCH 8/9] edit --- src/adapters/loading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/loading.py b/src/adapters/loading.py index 226ac5d74d..620b056d03 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -381,7 +381,7 @@ def _fix_backward_compat(self, config): and config["config"]["r"] != config["config"]["alpha"] ): logger.warning( - "Loading a LoRA trained using a faulty scaling implementation. Editing the configuration to make sure the adapter works as trained." + "Loading a LoRA trained using a faulty scaling implementation of a previous library version. Editing the configuration to make sure the adapter works as trained." "See https://github.com/adapter-hub/adapters/pull/770 for more." ) config["config"]["alpha"] = config["config"]["r"] From 56e9c9334296695f538497c68a9f5006a91cf7e4 Mon Sep 17 00:00:00 2001 From: calpt Date: Sun, 29 Dec 2024 22:20:05 +0000 Subject: [PATCH 9/9] style --- src/adapters/loading.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/loading.py b/src/adapters/loading.py index 620b056d03..69747e04cb 100644 --- a/src/adapters/loading.py +++ b/src/adapters/loading.py @@ -375,7 +375,7 @@ def _fix_backward_compat(self, config): MIN_VERSION = Version("1.1.0") version = config.get("version", "") - if version.startswith(ADAPTER_PREFIX) and Version(version[len(ADAPTER_PREFIX):]) < MIN_VERSION: + if version.startswith(ADAPTER_PREFIX) and Version(version[len(ADAPTER_PREFIX) :]) < MIN_VERSION: if ( config["config"].get("architecture", None) == "lora" and config["config"]["r"] != config["config"]["alpha"]