From b7d821fc2808283fa87d2e68a2dccb49339019b4 Mon Sep 17 00:00:00 2001 From: Benjamin Badger Date: Wed, 15 Jan 2025 12:22:29 -0500 Subject: [PATCH 1/3] pixel input assignment revoked --- src/transformers/models/mllama/modeling_mllama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/mllama/modeling_mllama.py b/src/transformers/models/mllama/modeling_mllama.py index cb9a20dadc67..f668f0d9ebd2 100644 --- a/src/transformers/models/mllama/modeling_mllama.py +++ b/src/transformers/models/mllama/modeling_mllama.py @@ -1537,7 +1537,7 @@ def forward( aspect_ratio_ids = aspect_ratio_ids.reshape(batch_size * num_concurrent_media, -1) # Patch embedding - patch_embeds = self.patch_embedding(pixel_values.to(self.dtype).to(self.device)) + patch_embeds = self.patch_embedding(pixel_values) hidden_state = patch_embeds.flatten(2).transpose(1, 2) # Tile embeddings From dedd1ee129d08a1b3f66b5b9f6ac681a5f40d05c Mon Sep 17 00:00:00 2001 From: Benjamin Badger Date: Fri, 31 Jan 2025 11:53:58 -0500 Subject: [PATCH 2/3] double send --- src/transformers/models/mllama/modeling_mllama.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/transformers/models/mllama/modeling_mllama.py b/src/transformers/models/mllama/modeling_mllama.py index f668f0d9ebd2..b4e81f686fef 100644 --- a/src/transformers/models/mllama/modeling_mllama.py +++ b/src/transformers/models/mllama/modeling_mllama.py @@ -1537,7 +1537,9 @@ def forward( aspect_ratio_ids = aspect_ratio_ids.reshape(batch_size * num_concurrent_media, -1) # Patch embedding - patch_embeds = self.patch_embedding(pixel_values) + target_dtype = self.patch_embedding.weight.dtype + target_device = self.patch_embedding.weight.device + patch_embeds = self.patch_embedding(pixel_values.to(target_dtype).to(target_device)) hidden_state = patch_embeds.flatten(2).transpose(1, 2) # Tile embeddings From 60b34ed54be8772466468bf0cb1599209bec0072 Mon Sep 17 00:00:00 2001 From: Benjamin Badger <54602201+blbadger@users.noreply.github.com> Date: Tue, 4 Feb 2025 18:41:22 -0500 Subject: [PATCH 3/3] Update src/transformers/models/mllama/modeling_mllama.py Co-authored-by: Pavel Iakubovskii --- src/transformers/models/mllama/modeling_mllama.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transformers/models/mllama/modeling_mllama.py b/src/transformers/models/mllama/modeling_mllama.py index 574654012a34..b69ea724c744 100644 --- a/src/transformers/models/mllama/modeling_mllama.py +++ b/src/transformers/models/mllama/modeling_mllama.py @@ -1540,7 +1540,7 @@ def forward( # Patch embedding target_dtype = self.patch_embedding.weight.dtype target_device = self.patch_embedding.weight.device - patch_embeds = self.patch_embedding(pixel_values.to(target_dtype).to(target_device)) + patch_embeds = self.patch_embedding(pixel_values.to(target_device, target_dtype)) hidden_state = patch_embeds.flatten(2).transpose(1, 2) # Tile embeddings