Add new stream: ad_insights_hourly_advertiser (singer-io#151)

* Add new stream: ads_insights_hourly_advertiser * Add new stream to tests * Get tests passing
sixcodes · Mar 17, 2023 · 2471b4f · 2471b4f
1 parent 48b542a
commit 2471b4f
Show file tree

Hide file tree

Showing 6 changed files with 325 additions and 4 deletions.
diff --git a/tap_facebook/__init__.py b/tap_facebook/__init__.py
@@ -56,6 +56,7 @@
     'ads_insights_platform_and_device',
     'ads_insights_region',
     'ads_insights_dma',
+    'ads_insights_hourly_advertiser',
     #'leads',
 ]
 
@@ -74,6 +75,7 @@
     'ads_insights_platform_and_device': START_DATE_KEY,
     'ads_insights_region': START_DATE_KEY,
     'ads_insights_dma': START_DATE_KEY,
+    'ads_insights_hourly_advertiser': START_DATE_KEY,
     'leads': CREATED_TIME_KEY,
 }
 
@@ -567,7 +569,7 @@ class AdsInsights(Stream):
     # these fields are not defined in the facebook_business library
     # Sending these fields is not allowed, but they are returned by the api
     invalid_insights_fields = ['impression_device', 'publisher_platform', 'platform_position',
-                               'age', 'gender', 'country', 'placement', 'region', 'dma']
+                               'age', 'gender', 'country', 'placement', 'region', 'dma', 'hourly_stats_aggregated_by_advertiser_time_zone']
     FACEBOOK_INSIGHTS_RETENTION_PERIOD = 37 # months
 
     # pylint: disable=no-member,unsubscriptable-object,attribute-defined-outside-init
@@ -692,6 +694,8 @@ def __iter__(self):
                             'primary-keys': ['region']},
     'ads_insights_dma': {"breakdowns": ['dma'],
                          "primary-keys": ['dma']},
+    'ads_insights_hourly_advertiser': {'breakdowns': ['hourly_stats_aggregated_by_advertiser_time_zone'],
+                                       "primary-keys": ['hourly_stats_aggregated_by_advertiser_time_zone']},
 }
 
 

diff --git a/tap_facebook/schemas/ads_insights_hourly_advertiser.json b/tap_facebook/schemas/ads_insights_hourly_advertiser.json
@@ -0,0 +1,309 @@
+{
+  "type": [
+    "null",
+    "object"
+  ],
+  "properties": {
+    "account_currency": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "account_id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "account_name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "action_values": {"$ref": "ads_action_stats.json"},
+    "actions": {"$ref": "ads_action_stats.json"},
+    "ad_id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "ad_name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "adset_id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "adset_name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "attribution_setting": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "buying_type": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "campaign_id": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "campaign_name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "canvas_avg_view_percent": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "canvas_avg_view_time": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "catalog_segment_value": {"$ref": "ads_action_stats.json"},
+    "clicks": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "conversion_rate_ranking": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "conversion_values": {"$ref": "ads_action_stats.json"},
+    "conversions": {"$ref": "ads_action_stats.json"},
+    "converted_product_quantity": {"$ref": "ads_action_stats.json"},
+    "converted_product_value": {"$ref": "ads_action_stats.json"},
+    "cost_per_action_type": {"$ref": "ads_action_stats.json"},
+    "cost_per_conversion": {"$ref": "ads_action_stats.json"},
+    "cost_per_estimated_ad_recallers": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "cost_per_inline_link_click": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "cost_per_inline_post_engagement": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "cost_per_outbound_click": {"$ref": "ads_action_stats.json"},
+    "cost_per_thruplay": {"$ref": "ads_action_stats.json"},
+    "cost_per_unique_action_type": {"$ref": "ads_action_stats.json"},
+    "cost_per_unique_click": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "cost_per_unique_inline_link_click": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "cost_per_unique_outbound_click": {"$ref": "ads_action_stats.json"},
+    "cpc": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "cpm": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "cpp": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "ctr": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "date_start": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "date_stop": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "engagement_rate_ranking": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "estimated_ad_recall_rate": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "estimated_ad_recallers": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "frequency": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "full_view_impressions": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "hourly_stats_aggregated_by_advertiser_time_zone": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "impressions": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "inline_link_click_ctr": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "inline_link_clicks": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "inline_post_engagement": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "instant_experience_clicks_to_open": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "instant_experience_clicks_to_start": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "instant_experience_outbound_clicks": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "mobile_app_purchase_roas": {"$ref": "ads_action_stats.json"},
+    "objective": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "optimization_goal": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "outbound_clicks": {"$ref": "ads_action_stats.json"},
+    "outbound_clicks_ctr": {"$ref": "ads_action_stats.json"},
+    "place_page_name": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "purchase_roas": {"$ref": "ads_action_stats.json"},
+    "qualifying_question_qualify_answer_rate": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "quality_ranking": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "reach": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "social_spend": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "spend": {
+      "type": [
+        "null",
+        "string"
+      ]
+    },
+    "website_ctr": {"$ref": "ads_action_stats.json"},
+    "website_purchase_roas": {"$ref": "ads_action_stats.json"}
+  }
+}
diff --git a/tests/base.py b/tests/base.py
@@ -119,6 +119,11 @@ def expected_metadata(self):
                 self.REPLICATION_METHOD: self.INCREMENTAL,
                 self.REPLICATION_KEYS: {"date_start"}
             },
+            "ads_insights_hourly_advertiser": {
+                self.PRIMARY_KEYS: {"hourly_stats_aggregated_by_advertiser_time_zone", "campaign_id", "adset_id", "ad_id", "date_start"},
+                self.REPLICATION_METHOD: self.INCREMENTAL,
+                self.REPLICATION_KEYS: {"date_start"}
+            },
             # "leads": {
             #     self.PRIMARY_KEYS: {"id"},
             #     self.REPLICATION_METHOD: self.INCREMENTAL,

diff --git a/tests/test_facebook_bookmarks.py b/tests/test_facebook_bookmarks.py
@@ -196,7 +196,7 @@ def bookmarks_test(self, expected_streams):
 
                         # Verify the second sync records respect the previous (simulated) bookmark value
                         replication_key_value = record.get(replication_key)
-                        if stream == 'ads_insights_age_and_gender': # BUG | https://stitchdata.atlassian.net/browse/SRCE-4873
+                        if stream in {'ads_insights_age_and_gender', 'ads_insights_hourly_advertiser'}: # BUG | https://stitchdata.atlassian.net/browse/SRCE-4873
                             replication_key_value = datetime.datetime.strftime(
                                 dateutil.parser.parse(replication_key_value),
                                 self.BOOKMARK_COMPARISON_FORMAT

diff --git a/tests/test_facebook_discovery.py b/tests/test_facebook_discovery.py
@@ -88,7 +88,7 @@ def test_run(self):
                 failing_with_no_replication_keys = {
                     'ads_insights_country', 'adsets', 'adcreative', 'ads', 'ads_insights_region',
                     'campaigns', 'ads_insights_age_and_gender', 'ads_insights_platform_and_device',
-                    'ads_insights_dma', 'ads_insights', 'leads'
+                    'ads_insights_dma', 'ads_insights', 'leads', 'ads_insights_hourly_advertiser'
                 }
                 if stream not in failing_with_no_replication_keys:  # BUG_1
                     # verify replication key(s) match expectations
@@ -105,7 +105,7 @@ def test_run(self):
                 failing_with_no_replication_method = {
                     'ads_insights_country', 'adsets', 'adcreative', 'ads', 'ads_insights_region',
                     'campaigns', 'ads_insights_age_and_gender', 'ads_insights_platform_and_device',
-                    'ads_insights_dma', 'ads_insights', 'leads'
+                    'ads_insights_dma', 'ads_insights', 'leads', 'ads_insights_hourly_advertiser'
                 }
                 if stream not in failing_with_no_replication_method:  # BUG_2
                     # verify the replication method matches our expectations