Add DAB-DETR Object detection/segmentation model

docs/source/en/model_doc/dab-detr.md

@@ -21,7 +21,7 @@ rendered properly in your Markdown viewer.
 The DAB-DETR model was proposed in [DAB-DETR: Dynamic Anchor Boxes are Better Queries for DETR](https://arxiv.org/abs/2201.12329) by Shilong Liu, Feng Li, Hao Zhang, Xiao Yang, Xianbiao Qi, Hang Su, Jun Zhu, Lei Zhang.
 DAB-DETR is an enhanced variant of Conditional DETR. It utilizes dynamically updated anchor boxes to provide both a reference query point (x, y) and a reference anchor size (w, h), improving cross-attention computation. This new approach achieves 45.7% AP when trained for 50 epochs with a single ResNet-50 model as the backbone.
 
-<img src="https://github.com/conditionedstimulus/hf_media/blob/main/dab_detr_convergence_plot.png"
+<img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/transformers/model_doc/dab_detr_convergence_plot.png"
 alt="drawing" width="600"/>
 
 The abstract from the paper is the following:
@@ -42,13 +42,52 @@ experiments to confirm our analysis and verify the effectiveness of our methods.
 This model was contributed by [davidhajdu](https://huggingface.co/davidhajdu).
 The original code can be found [here](https://github.com/IDEA-Research/DAB-DETR).
 
-There are three ways to instantiate a DAB-DETR model (depending on what you prefer):
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+```python
+import torch
+import requests
+
+from PIL import Image
+from transformers import AutoModelForObjectDetection, AutoImageProcessor
+
+url = 'http://images.cocodataset.org/val2017/000000039769.jpg' 
+image = Image.open(requests.get(url, stream=True).raw)
+
+image_processor = AutoImageProcessor.from_pretrained("IDEA-Research/dab-detr-resnet-50")
+model = AutoModelForObjectDetection.from_pretrained("IDEA-Research/dab-detr-resnet-50")
+
+inputs = image_processor(images=image, return_tensors="pt")
+
+with torch.no_grad():
+    outputs = model(**inputs)
+
+results = image_processor.post_process_object_detection(outputs, target_sizes=torch.tensor([image.size[::-1]]), threshold=0.3)
+
+for result in results:
+    for score, label_id, box in zip(result["scores"], result["labels"], result["boxes"]):
+        score, label = score.item(), label_id.item()
+        box = [round(i, 2) for i in box.tolist()]
+        print(f"{model.config.id2label[label]}: {score:.2f} {box}")
+```
+This should output
+```
+cat: 0.87 [14.7, 49.39, 320.52, 469.28]
+remote: 0.86 [41.08, 72.37, 173.39, 117.2]
+cat: 0.86 [344.45, 19.43, 639.85, 367.86]
+remote: 0.61 [334.27, 75.93, 367.92, 188.81]
+couch: 0.59 [-0.04, 1.34, 639.9, 477.09]
+```
+
+There are three other ways to instantiate a DAB-DETR model (depending on what you prefer):
 
 Option 1: Instantiate DAB-DETR with pre-trained weights for entire model
 ```py
 >>> from transformers import DabDetrForObjectDetection
 
->>> model = DabDetrForObjectDetection.from_pretrained("IDEA-Research/dab_detr_resnet50")
+>>> model = DabDetrForObjectDetection.from_pretrained("IDEA-Research/dab-detr-resnet-50")
 ```
 
 Option 2: Instantiate DAB-DETR with randomly initialized weights for Transformer, but pre-trained weights for backbone

src/transformers/models/dab_detr/convert_dab_detr_original_pytorch_checkpoint_to_pytorch.py

@@ -44,7 +44,7 @@
     r"transformer\.decoder\.ref_point_head\.layers\.(\d+)\.(bias|weight)": r"decoder.ref_point_head.layers.\1.\2",
     r"transformer\.decoder\.ref_anchor_head\.layers\.(\d+)\.(bias|weight)": r"decoder.ref_anchor_head.layers.\1.\2",
     r"transformer\.decoder\.query_scale\.layers\.(\d+)\.(bias|weight)": r"decoder.query_scale.layers.\1.\2",
-    r"transformer\.decoder\.layers\.0\.ca_qpos_proj\.(bias|weight)": r"decoder.layers.0.layer.1.cross_attn_query_pos_proj.\1",
+    r"transformer\.decoder\.layers\.0\.ca_qpos_proj\.(bias|weight)": r"decoder.layers.0.cross_attn.cross_attn_query_pos_proj.\1",
     # encoder layers: output projection, 2 feedforward neural networks and 2 layernorms + activation function
     # output projection
     r"transformer\.encoder\.layers\.(\d+)\.self_attn\.out_proj\.(bias|weight)": r"encoder.layers.\1.self_attn.out_proj.\2",
@@ -59,30 +59,30 @@
     r"transformer\.encoder\.layers\.(\d+)\.activation\.weight": r"encoder.layers.\1.activation_fn.weight",
     #########################################################################################################################################
     # decoder layers: 2 times output projection, 2 feedforward neural networks and 3 layernorms + activiation function weight
-    r"transformer\.decoder\.layers\.(\d+)\.self_attn\.out_proj\.(bias|weight)": r"decoder.layers.\1.layer.0.self_attn.output_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.cross_attn\.out_proj\.(bias|weight)": r"decoder.layers.\1.layer.1.cross_attn.output_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.self_attn\.out_proj\.(bias|weight)": r"decoder.layers.\1.self_attn.self_attn.output_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.cross_attn\.out_proj\.(bias|weight)": r"decoder.layers.\1.cross_attn.cross_attn.output_proj.\2",
     # FFNs
-    r"transformer\.decoder\.layers\.(\d+)\.linear(\d)\.(bias|weight)": r"decoder.layers.\1.layer.2.fc\2.\3",
+    r"transformer\.decoder\.layers\.(\d+)\.linear(\d)\.(bias|weight)": r"decoder.layers.\1.mlp.fc\2.\3",
     # nm1
-    r"transformer\.decoder\.layers\.(\d+)\.norm1\.(bias|weight)": r"decoder.layers.\1.layer.0.self_attn_layer_norm.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.norm1\.(bias|weight)": r"decoder.layers.\1.self_attn.self_attn_layer_norm.\2",
     # nm2
-    r"transformer\.decoder\.layers\.(\d+)\.norm2\.(bias|weight)": r"decoder.layers.\1.layer.1.cross_attn_layer_norm.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.norm2\.(bias|weight)": r"decoder.layers.\1.cross_attn.cross_attn_layer_norm.\2",
     # nm3
-    r"transformer\.decoder\.layers\.(\d+)\.norm3\.(bias|weight)": r"decoder.layers.\1.layer.2.final_layer_norm.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.norm3\.(bias|weight)": r"decoder.layers.\1.mlp.final_layer_norm.\2",
     # activation function weight
-    r"transformer\.decoder\.layers\.(\d+)\.activation\.weight": r"decoder.layers.\1.layer.2.activation_fn.weight",
+    r"transformer\.decoder\.layers\.(\d+)\.activation\.weight": r"decoder.layers.\1.mlp.activation_fn.weight",
     # q, k, v projections and biases in self-attention in decoder
-    r"transformer\.decoder\.layers\.(\d+)\.sa_qcontent_proj\.(bias|weight)": r"decoder.layers.\1.layer.0.self_attn_query_content_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.sa_kcontent_proj\.(bias|weight)": r"decoder.layers.\1.layer.0.self_attn_key_content_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.sa_qpos_proj\.(bias|weight)": r"decoder.layers.\1.layer.0.self_attn_query_pos_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.sa_kpos_proj\.(bias|weight)": r"decoder.layers.\1.layer.0.self_attn_key_pos_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.sa_v_proj\.(bias|weight)": r"decoder.layers.\1.layer.0.self_attn_value_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.sa_qcontent_proj\.(bias|weight)": r"decoder.layers.\1.self_attn.self_attn_query_content_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.sa_kcontent_proj\.(bias|weight)": r"decoder.layers.\1.self_attn.self_attn_key_content_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.sa_qpos_proj\.(bias|weight)": r"decoder.layers.\1.self_attn.self_attn_query_pos_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.sa_kpos_proj\.(bias|weight)": r"decoder.layers.\1.self_attn.self_attn_key_pos_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.sa_v_proj\.(bias|weight)": r"decoder.layers.\1.self_attn.self_attn_value_proj.\2",
     # q, k, v projections in cross-attention in decoder
-    r"transformer\.decoder\.layers\.(\d+)\.ca_qcontent_proj\.(bias|weight)": r"decoder.layers.\1.layer.1.cross_attn_query_content_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.ca_kcontent_proj\.(bias|weight)": r"decoder.layers.\1.layer.1.cross_attn_key_content_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.ca_kpos_proj\.(bias|weight)": r"decoder.layers.\1.layer.1.cross_attn_key_pos_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.ca_v_proj\.(bias|weight)": r"decoder.layers.\1.layer.1.cross_attn_value_proj.\2",
-    r"transformer\.decoder\.layers\.(\d+)\.ca_qpos_sine_proj\.(bias|weight)": r"decoder.layers.\1.layer.1.cross_attn_query_pos_sine_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.ca_qcontent_proj\.(bias|weight)": r"decoder.layers.\1.cross_attn.cross_attn_query_content_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.ca_kcontent_proj\.(bias|weight)": r"decoder.layers.\1.cross_attn.cross_attn_key_content_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.ca_kpos_proj\.(bias|weight)": r"decoder.layers.\1.cross_attn.cross_attn_key_pos_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.ca_v_proj\.(bias|weight)": r"decoder.layers.\1.cross_attn.cross_attn_value_proj.\2",
+    r"transformer\.decoder\.layers\.(\d+)\.ca_qpos_sine_proj\.(bias|weight)": r"decoder.layers.\1.cross_attn.cross_attn_query_pos_sine_proj.\2",
 }
 
 

src/transformers/models/dab_detr/modeling_dab_detr.py

@@ -772,10 +772,9 @@ def forward(
 class DabDetrDecoderLayer(nn.Module):
     def __init__(self, config: DabDetrConfig, is_first: bool = False):
         super().__init__()
-        self.layer = nn.ModuleList()
-        self.layer.append(DabDetrDecoderLayerSelfAttention(config))
-        self.layer.append(DabDetrDecoderLayerCrossAttention(config, is_first))
-        self.layer.append(DabDetrDecoderLayerFFN(config))
+        self.self_attn = DabDetrDecoderLayerSelfAttention(config)
+        self.cross_attn = DabDetrDecoderLayerCrossAttention(config, is_first)
+        self.mlp = DabDetrDecoderLayerFFN(config)
 
     def forward(
         self,
@@ -810,14 +809,14 @@ def forward(
                 returned tensors for more detail.
 
         """
-        hidden_states, self_attn_weights = self.layer[0](
+        hidden_states, self_attn_weights = self.self_attn(
             hidden_states=hidden_states,
             query_position_embeddings=query_position_embeddings,
             attention_mask=attention_mask,
             output_attentions=output_attentions,
         )
 
-        hidden_states, cross_attn_weights = self.layer[1](
+        hidden_states, cross_attn_weights = self.cross_attn(
             hidden_states=hidden_states,
             encoder_hidden_states=encoder_hidden_states,
             query_position_embeddings=query_position_embeddings,
@@ -827,7 +826,7 @@ def forward(
             output_attentions=output_attentions,
         )
 
-        hidden_states = self.layer[2](hidden_states=hidden_states)
+        hidden_states = self.mlp(hidden_states=hidden_states)
 
         outputs = (hidden_states,)
 
@@ -973,6 +972,7 @@ def __init__(self, config: DabDetrConfig):
         self.query_scale = DabDetrMLP(config.hidden_size, config.hidden_size, config.hidden_size, 2)
         self.layers = nn.ModuleList([DabDetrEncoderLayer(config) for _ in range(config.encoder_layers)])
         self.norm = nn.LayerNorm(config.hidden_size) if config.normalize_before else None
+        self.gradient_checkpointing = False
 
         # Initialize weights and apply final processing
         self.post_init()
@@ -1032,14 +1032,24 @@ def forward(
                 encoder_states = encoder_states + (hidden_states,)
             # pos scaler
             pos_scales = self.query_scale(hidden_states)
-            scaled_object_queries = object_queries * pos_scales
             # we add object_queries * pos_scaler as extra input to the encoder_layer
-            layer_outputs = encoder_layer(
-                hidden_states,
-                attention_mask=attention_mask,
-                object_queries=scaled_object_queries,
-                output_attentions=output_attentions,
-            )
+            scaled_object_queries = object_queries * pos_scales
+
+            if self.gradient_checkpointing and self.training:
+                layer_outputs = self._gradient_checkpointing_func(
+                    encoder_layer.__call__,
+                    hidden_states,
+                    attention_mask,
+                    scaled_object_queries,
+                    output_attentions,
+                )
+            else:
+                layer_outputs = encoder_layer(
+                    hidden_states,
+                    attention_mask=attention_mask,
+                    object_queries=scaled_object_queries,
+                    output_attentions=output_attentions,
+                )
 
             hidden_states = layer_outputs[0]
 
@@ -1178,10 +1188,14 @@ def forward(
             # apply transformation
             query_sine_embed = query_sine_embed[..., : self.hidden_size] * pos_transformation
 
-            # modulated HW attentions
-            refHW_cond = self.ref_anchor_head(hidden_states).sigmoid()  # nq, bs, 2
-            query_sine_embed[..., self.hidden_size // 2 :] *= (refHW_cond[..., 0] / obj_center[..., 2]).unsqueeze(-1)
-            query_sine_embed[..., : self.hidden_size // 2] *= (refHW_cond[..., 1] / obj_center[..., 3]).unsqueeze(-1)
+            # modulated Height Width attentions
+            reference_anchor_size = self.ref_anchor_head(hidden_states).sigmoid()  # nq, bs, 2
+            query_sine_embed[..., self.hidden_size // 2 :] *= (
+                reference_anchor_size[..., 0] / obj_center[..., 2]
+            ).unsqueeze(-1)
+            query_sine_embed[..., : self.hidden_size // 2] *= (
+                reference_anchor_size[..., 1] / obj_center[..., 3]
+            ).unsqueeze(-1)
 
             if self.gradient_checkpointing and self.training:
                 layer_outputs = self._gradient_checkpointing_func(
@@ -1227,10 +1241,10 @@ def forward(
                 if encoder_hidden_states is not None:
                     all_cross_attentions += (layer_outputs[2],)
 
-        if self.layernorm is not None:
-            hidden_states = self.layernorm(hidden_states)
-            intermediate.pop()
-            intermediate.append(hidden_states)
+        # Layer normalization on hidden states and add it to the intermediate list
+        hidden_states = self.layernorm(hidden_states)
+        intermediate.pop()
+        intermediate.append(hidden_states)
 
         if output_hidden_states:
             all_hidden_states += (hidden_states,)
@@ -1302,7 +1316,7 @@ def __init__(self, config: DabDetrConfig):
 
         self.num_patterns = config.num_patterns
         if not isinstance(self.num_patterns, int):
-            Warning("num_patterns should be int but {}".format(type(self.num_patterns)))
+            logger.warning("num_patterns should be int but {}".format(type(self.num_patterns)))
             self.num_patterns = 0
         if self.num_patterns > 0:
             self.patterns = nn.Embedding(self.num_patterns, self.hidden_size)
@@ -1609,8 +1623,8 @@ def forward(
         >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg"
         >>> image = Image.open(requests.get(url, stream=True).raw)
 
-        >>> image_processor = AutoImageProcessor.from_pretrained("IDEA-Research/dab_detr-base")
-        >>> model = AutoModelForObjectDetection.from_pretrained("IDEA-Research/dab_detr-base")
+        >>> image_processor = AutoImageProcessor.from_pretrained("IDEA-Research/dab-detr-resnet-50")
+        >>> model = AutoModelForObjectDetection.from_pretrained("IDEA-Research/dab-detr-resnet-50")
 
         >>> inputs = image_processor(images=image, return_tensors="pt")
 
@@ -1658,9 +1672,9 @@ def forward(
         logits = self.class_embed(intermediate_hidden_states[-1])
 
         reference_before_sigmoid = inverse_sigmoid(reference_points)
-        tmp = self.bbox_predictor(intermediate_hidden_states)
-        tmp[..., : self.query_dim] += reference_before_sigmoid
-        outputs_coord = tmp.sigmoid()
+        bbox_with_refinement = self.bbox_predictor(intermediate_hidden_states)
+        bbox_with_refinement[..., : self.query_dim] += reference_before_sigmoid
+        outputs_coord = bbox_with_refinement.sigmoid()
 
         pred_boxes = outputs_coord[-1]
 

tests/models/dab_detr/test_modeling_dab_detr.py

@@ -295,10 +295,11 @@ def recursive_check(tuple_object, dict_object):
                     elif tuple_object is None:
                         return
                     else:
-                        self.assertTrue(
-                            torch.allclose(
-                                set_nan_tensor_to_zero(tuple_object), set_nan_tensor_to_zero(dict_object), atol=1e-5
-                            ),
+                        torch.testing.assert_close(
+                            set_nan_tensor_to_zero(tuple_object),
+                            set_nan_tensor_to_zero(dict_object),
+                            atol=1e-5,
+                            rtol=1e-5,
                             msg=(
                                 "Tuple and dict output are not equal. Difference:"
                                 f" {torch.max(torch.abs(tuple_object - dict_object))}. Tuple has `nan`:"
@@ -735,8 +736,11 @@ def test_initialization(self):
                     # Modifed from RT-DETR
                     elif "class_embed" in name and "bias" in name:
                         bias_tensor = torch.full_like(param.data, bias_value)
-                        self.assertTrue(
-                            torch.allclose(param.data, bias_tensor, atol=1e-4),
+                        torch.testing.assert_close(
+                            param.data,
+                            bias_tensor,
+                            atol=1e-4,
+                            rtol=1e-4,
                             msg=f"Parameter {name} of model {model_class} seems not properly initialized",
                         )
                     elif "activation_fn" in name and config.activation_function == "prelu":
@@ -793,7 +797,7 @@ def test_inference_no_head(self):
         expected_slice = torch.tensor(
             [[-0.4879, -0.2594, 0.4524], [-0.4997, -0.4258, 0.4329], [-0.8220, -0.4996, 0.0577]]
         ).to(torch_device)
-        self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3], expected_slice, atol=2e-4))
+        torch.testing.assert_close(outputs.last_hidden_state[0, :3, :3], expected_slice, atol=2e-4, rtol=2e-4)
 
     def test_inference_object_detection_head(self):
         model = DabDetrForObjectDetection.from_pretrained(CHECKPOINT).to(torch_device)
@@ -812,14 +816,14 @@ def test_inference_object_detection_head(self):
         expected_slice_logits = torch.tensor(
             [[-10.1765, -5.5243, -8.9324], [-9.8138, -5.6721, -7.5161], [-10.3054, -5.6081, -8.5931]]
         ).to(torch_device)
-        self.assertTrue(torch.allclose(outputs.logits[0, :3, :3], expected_slice_logits, atol=3e-4))
+        torch.testing.assert_close(outputs.logits[0, :3, :3], expected_slice_logits, atol=3e-4, rtol=3e-4)
 
         expected_shape_boxes = torch.Size((1, model.config.num_queries, 4))
         self.assertEqual(outputs.pred_boxes.shape, expected_shape_boxes)
         expected_slice_boxes = torch.tensor(
             [[0.3708, 0.3000, 0.2753], [0.5211, 0.6125, 0.9495], [0.2897, 0.6730, 0.5459]]
         ).to(torch_device)
-        self.assertTrue(torch.allclose(outputs.pred_boxes[0, :3, :3], expected_slice_boxes, atol=1e-4))
+        torch.testing.assert_close(outputs.pred_boxes[0, :3, :3], expected_slice_boxes, atol=1e-4, rtol=1e-4)
 
         # verify postprocessing
         results = image_processor.post_process_object_detection(