Skip to content

Commit

Permalink
Add support for custom inputs and batched inputs in ProcessorTesterMi…
Browse files Browse the repository at this point in the history 
…xin (#33711)

* add support for custom inputs and batched inputs in ProcessorTesterMixin

* Fix batch_size behavior ProcessorTesterMixin

* Change format prepare inputs batched

* Remove override test pixtral processor

* Remove unnecessary tests and cleanup after new prepare_inputs functions

* Fix instructBlipVideo image processor
  • Loading branch information
@yonigozlan
yonigozlan authored Oct 1, 2024
1 parent 1baa088 commit 61ac161
Show file tree
Hide file tree
Showing 8 changed files with 95 additions and 269 deletions.
7 changes: 5 additions & 2 deletions src/transformers/models/instructblipvideo/image_processing_instructblipvideo.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,11 @@ def make_batched_videos(videos) -> List[VideoInput]:
elif len(videos[0].shape) == 4:
return [list(video) for video in videos]

elif is_valid_image(videos) and len(videos.shape) == 4:
return [list(videos)]
elif is_valid_image(videos):
if isinstance(videos, PIL.Image.Image):
return [[videos]]
elif len(videos.shape) == 4:
return [list(videos)]

raise ValueError(f"Could not make batched video from {videos}")

Expand Down
12 changes: 6 additions & 6 deletions tests/models/fuyu/test_processing_fuyu.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ def test_kwargs_overrides_default_tokenizer_kwargs(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer"
input_str = self.prepare_text_inputs()
# Fuyu uses tokenizer kwargs only when image is None.
image_input = None

Expand Down Expand Up @@ -218,7 +218,7 @@ def test_tokenizer_defaults_preserved_by_kwargs(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer"
input_str = self.prepare_text_inputs()
# Fuyu uses tokenizer kwargs only when image is None.
image_input = None

Expand All @@ -237,7 +237,7 @@ def test_structured_kwargs_nested(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
input_str = self.prepare_text_inputs()
# Fuyu uses tokenizer kwargs only when image is None.
image_input = None

Expand All @@ -264,7 +264,7 @@ def test_structured_kwargs_nested_from_dict(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer"
input_str = self.prepare_text_inputs()
# Fuyu uses tokenizer kwargs only when image is None.
image_input = None

Expand All @@ -290,7 +290,7 @@ def test_unstructured_kwargs(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
input_str = self.prepare_text_inputs()
# Fuyu uses tokenizer kwargs only when image is None.
image_input = None
inputs = processor(
Expand All @@ -315,7 +315,7 @@ def test_unstructured_kwargs_batched(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = ["lower newer", "upper older longer string"]
input_str = self.prepare_text_inputs(batch_size=2)
# Fuyu uses tokenizer kwargs only when image is None.
image_input = None
inputs = processor(
Expand Down
70 changes: 28 additions & 42 deletions tests/models/idefics3/test_processing_idefics3.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import tempfile
import unittest
from io import BytesIO
from typing import Optional

import numpy as np
import requests
Expand Down Expand Up @@ -284,44 +285,29 @@ def test_apply_chat_template(self):
)
self.assertEqual(rendered, expected_rendered)

@require_torch
@require_vision
def test_image_processor_defaults_preserved_by_image_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
image_processor = self.get_component("image_processor")
tokenizer = self.get_component("tokenizer", max_length=117)
# Override as Idefics3Processor needs image tokens in prompts
def prepare_text_inputs(self, batch_size: Optional[int] = None):
if batch_size is None:
return "lower newer <image>"

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer <image>"
image_input = self.prepare_image_inputs()
if batch_size < 1:
raise ValueError("batch_size must be greater than 0")

inputs = processor(text=input_str, images=image_input)
self.assertEqual(len(inputs["pixel_values"][0][0]), 3)
self.assertEqual(len(inputs["pixel_values"][0][0][0]), 364) # crop size doesn't affect our image processor

@require_torch
@require_vision
def test_kwargs_overrides_default_image_processor_kwargs(self):
if "image_processor" not in self.processor_class.attributes:
self.skipTest(f"image_processor attribute not present in {self.processor_class}")
image_processor = self.get_component(
"image_processor", max_image_size={"longest_edge": 32}, size={"longest_edge": 32}
if batch_size == 1:
return ["lower newer <image>"]
return ["lower newer <image>", "<image> upper older longer string"] + ["<image> lower newer"] * (
batch_size - 2
)
tokenizer = self.get_component("tokenizer", max_length=117, padding="max_length")

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor, image_seq_len=2)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer <image>"
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input)
self.assertEqual(len(inputs["pixel_values"][0][0]), 3)
self.assertEqual(len(inputs["pixel_values"][0][0][0]), 32)
self.assertEqual(len(inputs["input_ids"][0]), 117)
# Override as Idefics3Processor needs nested images to work properly with batched inputs
@require_vision
def prepare_image_inputs(self, batch_size: Optional[int] = None):
"""This function prepares a list of PIL images for testing"""
if batch_size is None:
return super().prepare_image_inputs()
if batch_size < 1:
raise ValueError("batch_size must be greater than 0")
return [[super().prepare_image_inputs()]] * batch_size

@require_vision
@require_torch
Expand All @@ -333,7 +319,7 @@ def test_kwargs_overrides_default_tokenizer_kwargs(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer<image>"
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input, return_tensors="pt", max_length=30)
Expand All @@ -350,7 +336,7 @@ def test_structured_kwargs_nested(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer<image>"
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()

# Define the kwargs for each modality
Expand Down Expand Up @@ -378,7 +364,7 @@ def test_structured_kwargs_nested_from_dict(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer<image>"
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()

# Define the kwargs for each modality
Expand All @@ -402,7 +388,7 @@ def test_tokenizer_defaults_preserved_by_kwargs(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer<image>"
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()

inputs = processor(text=input_str, images=image_input, return_tensors="pt")
Expand All @@ -419,11 +405,11 @@ def test_unstructured_kwargs_batched(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = ["<image>lower newer", "<image>upper older longer string"]
image_input = self.prepare_image_inputs()
input_str = self.prepare_text_inputs(batch_size=2)
image_input = self.prepare_image_inputs(batch_size=2)
inputs = processor(
text=input_str,
images=[image_input, image_input],
images=image_input,
return_tensors="pt",
padding="longest",
max_length=76,
Expand All @@ -446,7 +432,7 @@ def test_unstructured_kwargs(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer<image>"
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()
inputs = processor(
text=input_str,
Expand Down
12 changes: 6 additions & 6 deletions tests/models/kosmos2/test_processor_kosmos2.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ def test_kwargs_overrides_default_tokenizer_kwargs(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer"
input_str = self.prepare_text_inputs()
# set image input to None
image_input = None

Expand All @@ -525,7 +525,7 @@ def test_structured_kwargs_nested(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()

# Define the kwargs for each modality
Expand All @@ -551,7 +551,7 @@ def test_structured_kwargs_nested_from_dict(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer"
input_str = self.prepare_text_inputs()
image_input = self.prepare_image_inputs()

# Define the kwargs for each modality
Expand All @@ -574,7 +574,7 @@ def test_tokenizer_defaults_preserved_by_kwargs(self):

processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)
input_str = "lower newer"
input_str = self.prepare_text_inputs()
# set image input to None
image_input = None

Expand All @@ -593,7 +593,7 @@ def test_unstructured_kwargs(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = "lower newer"
input_str = self.prepare_text_inputs()
# set image input to None
image_input = None
inputs = processor(
Expand All @@ -618,7 +618,7 @@ def test_unstructured_kwargs_batched(self):
processor = self.processor_class(tokenizer=tokenizer, image_processor=image_processor)
self.skip_processor_without_typed_kwargs(processor)

input_str = ["lower newer", "upper older longer string"]
input_str = self.prepare_text_inputs(batch_size=2)
# set image input to None
image_input = None
inputs = processor(
Expand Down
Loading

0 comments on commit 61ac161

Please sign in to comment.