From 3d96130414cb3a198ba980cffd502bfc0279881c Mon Sep 17 00:00:00 2001 From: julian fong Date: Wed, 27 Nov 2024 21:10:56 -0500 Subject: [PATCH 1/2] initial commit --- src/adapters/methods/modeling.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/adapters/methods/modeling.py b/src/adapters/methods/modeling.py index d1e19c257..ec561007c 100644 --- a/src/adapters/methods/modeling.py +++ b/src/adapters/methods/modeling.py @@ -123,9 +123,11 @@ def __init__( self.gate.apply(self.init_bert_weights) elif config["init_weights"] == "mam_adapter": with torch.no_grad(): - nn.init.kaiming_uniform_(self.adapter_down[0].weight, a=math.sqrt(5)) + for layer in self.adapter_down: + if isinstance(layer, nn.Linear) or isinstance(layer, PHMLayer): + nn.init.kaiming_uniform_(layer.weight, a=math.sqrt(5)) + nn.init.zeros_(layer.bias) nn.init.zeros_(self.adapter_up.weight) - nn.init.zeros_(self.adapter_down[0].bias) nn.init.zeros_(self.adapter_up.bias) if self.use_gating: self.gate.apply(self.init_bert_weights) From ddfd8bf7dd9015d59fcc3a16a8e4f4419fd4a224 Mon Sep 17 00:00:00 2001 From: Julian Fong <44014224+julian-fong@users.noreply.github.com> Date: Wed, 27 Nov 2024 22:12:10 -0500 Subject: [PATCH 2/2] Update ViT_AdapterPlus_FineTuning.ipynb --- notebooks/ViT_AdapterPlus_FineTuning.ipynb | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/notebooks/ViT_AdapterPlus_FineTuning.ipynb b/notebooks/ViT_AdapterPlus_FineTuning.ipynb index 8dfbcd341..1cf549ea7 100644 --- a/notebooks/ViT_AdapterPlus_FineTuning.ipynb +++ b/notebooks/ViT_AdapterPlus_FineTuning.ipynb @@ -302,25 +302,6 @@ ")" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "trainer = AdapterTrainer(\n", - " model=model,\n", - " args=training_args,\n", - " data_collator=data_collator,\n", - " train_dataset=train_dataset,\n", - " eval_dataset=eval_dataset,\n", - " tokenizer=processor,\n", - " compute_metrics = compute_metrics\n", - ")\n", - "\n", - "trainer.train()" - ] - }, { "cell_type": "code", "execution_count": null,