TDL-6148: Catch and retry Attribute error and TDL-13267: Fix AdCreative requests that are not retrying

.circleci/config.yml

@@ -53,7 +53,13 @@ jobs:
           name: 'Run Unit Tests'
           command: |
             source /usr/local/share/virtualenvs/tap-facebook/bin/activate
-            nosetests tests/unittests
+            pip install nose coverage
+            nosetests --with-coverage --cover-erase --cover-package=tap_facebook --cover-html-dir=htmlcov tests/unittests
+            coverage html
+      - store_test_results:
+          path: test_output/report.xml
+      - store_artifacts:
+          path: htmlcov
       - slack/notify-on-failure:
           only_for_branches: master
   run_integration_tests:

tap_facebook/__init__.py

@@ -153,6 +153,8 @@ def should_retry_api_error(exception):
             return True
         elif isinstance(exception, TypeError) and str(exception) == "string indices must be integers":
             return True
+        elif isinstance(exception, AttributeError):
+            return True
         return False
 
     return backoff.on_exception(
@@ -241,6 +243,7 @@ class AdCreative(Stream):
     doc: https://developers.facebook.com/docs/marketing-api/reference/adgroup/adcreatives/
     '''
 
+    @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
     def sync_batches(self, stream_objects):
         refs = load_shared_schema_refs()
         schema = singer.resolve_schema_references(self.catalog_entry.schema.to_dict(), refs)
@@ -269,7 +272,7 @@ def sync_batches(self, stream_objects):
 
     key_properties = ['id']
 
-    @retry_pattern(backoff.expo, (FacebookRequestError, TypeError), max_tries=5, factor=5)
+    @retry_pattern(backoff.expo, (FacebookRequestError, TypeError, AttributeError), max_tries=5, factor=5)
     def get_adcreatives(self):
         return self.account.get_ad_creatives(params={'limit': RESULT_RETURN_LIMIT})
 
@@ -285,7 +288,7 @@ class Ads(IncrementalStream):
 
     key_properties = ['id', 'updated_time']
 
-    @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+    @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
     def _call_get_ads(self, params):
         """
         This is necessary because the functions that call this endpoint return
@@ -310,7 +313,7 @@ def do_request_multiple():
                 filt_ads = self._call_get_ads(params)
                 yield filt_ads
 
-        @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+        @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
         def prepare_record(ad):
             return ad.api_get(fields=self.fields()).export_all_data()
 
@@ -329,7 +332,7 @@ class AdSets(IncrementalStream):
 
     key_properties = ['id', 'updated_time']
 
-    @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+    @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
     def _call_get_ad_sets(self, params):
         """
         This is necessary because the functions that call this endpoint return
@@ -354,7 +357,7 @@ def do_request_multiple():
                 filt_adsets = self._call_get_ad_sets(params)
                 yield filt_adsets
 
-        @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+        @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
         def prepare_record(ad_set):
             return ad_set.api_get(fields=self.fields()).export_all_data()
 
@@ -370,7 +373,7 @@ class Campaigns(IncrementalStream):
 
     key_properties = ['id']
 
-    @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+    @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
     def _call_get_campaigns(self, params):
         """
         This is necessary because the functions that call this endpoint return
@@ -400,7 +403,7 @@ def do_request_multiple():
                 filt_campaigns = self._call_get_campaigns(params)
                 yield filt_campaigns
 
-        @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+        @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
         def prepare_record(campaign):
             """If campaign.ads is selected, make the request and insert the data here"""
             campaign_out = campaign.api_get(fields=fields).export_all_data()
@@ -436,6 +439,7 @@ def compare_lead_created_times(self, leadA, leadB):
         else:
             return leadA
 
+    @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
     def sync_batches(self, stream_objects):
         refs = load_shared_schema_refs()
         schema = singer.resolve_schema_references(self.catalog_entry.schema.to_dict(), refs)
@@ -469,12 +473,12 @@ def sync_batches(self, stream_objects):
         api_batch.execute()
         return str(pendulum.parse(latest_lead[self.replication_key]))
 
-    @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+    @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
     def get_ads(self):
         params = {'limit': RESULT_RETURN_LIMIT}
         yield from self.account.get_ads(params=params)
 
-    @retry_pattern(backoff.expo, FacebookRequestError, max_tries=5, factor=5)
+    @retry_pattern(backoff.expo, (FacebookRequestError, AttributeError), max_tries=5, factor=5)
     def get_leads(self, ads, start_time, previous_start_time):
         start_time = int(start_time.timestamp()) # Get unix timestamp
         params = {'limit': RESULT_RETURN_LIMIT,
@@ -615,7 +619,7 @@ def job_params(self):
             }
             buffered_start_date = buffered_start_date.add(days=1)
 
-    @retry_pattern(backoff.expo, (FacebookRequestError, InsightsJobTimeout, FacebookBadObjectError, TypeError), max_tries=5, factor=5)
+    @retry_pattern(backoff.expo, (FacebookRequestError, InsightsJobTimeout, FacebookBadObjectError, TypeError, AttributeError), max_tries=5, factor=5)
     def run_job(self, params):
         LOGGER.info('Starting adsinsights job with params %s', params)
         job = self.account.get_insights( # pylint: disable=no-member

tests/unittests/test_attribute_error_retry.py

@@ -0,0 +1,193 @@
+import unittest
+from unittest.mock import Mock
+from unittest import mock
+from tap_facebook import AdCreative, Ads, AdSets, Campaigns, AdsInsights, Leads
+
+@mock.patch("time.sleep")
+class TestAttributErrorBackoff(unittest.TestCase):
+    """A set of unit tests to ensure that requests are retrying properly for AttributeError Error"""
+    def test_get_adcreatives(self, mocked_sleep):
+        """ 
+            AdCreative.get_adcreatives calls a `facebook_business` method,`get_ad_creatives()`, to get a batch of ad creatives. 
+            We mock this method to raise a `AttributeError` and expect the tap to retry this that function up to 5 times,
+            which is the current hard coded `max_tries` value.
+        """
+
+        # Mock get_ad_creatives function to throw AttributeError exception
+        mocked_account = Mock()
+        mocked_account.get_ad_creatives = Mock()
+        mocked_account.get_ad_creatives.side_effect = AttributeError
+
+        # Call get_adcreatives() function of AdCreatives and verify AttributeError is raised
+        ad_creative_object = AdCreative('', mocked_account, '', '')
+        with self.assertRaises(AttributeError):
+            ad_creative_object.get_adcreatives()
+
+        # verify get_ad_creatives() is called 5 times as max 5 reties provided for function
+        self.assertEquals(mocked_account.get_ad_creatives.call_count, 5)
+
+    def test__call_get_ads(self, mocked_sleep):
+        """ 
+            Ads._call_get_ads calls a `facebook_business` method,`get_ads()`, to get a batch of ads. 
+            We mock this method to raise a `AttributeError` and expect the tap to retry this that function up to 5 times,
+            which is the current hard coded `max_tries` value.
+        """
+
+        # Mock get_ads function to throw AttributeError exception
+        mocked_account = Mock()
+        mocked_account.get_ads = Mock()
+        mocked_account.get_ads.side_effect = AttributeError
+
+        # Call _call_get_ads() function of Ads and verify AttributeError is raised
+        ad_object = Ads('', mocked_account, '', '', '')
+        with self.assertRaises(AttributeError):
+            ad_object._call_get_ads('test')
+
+        # verify get_ads() is called 5 times as max 5 reties provided for function
+        self.assertEquals(mocked_account.get_ads.call_count, 5)
+
+    @mock.patch("pendulum.parse")
+    def test_ad_prepare_record(self, mocked_parse, mocked_sleep):
+        """ 
+            __iter__ of Ads calls a function _iterate which calls a nested prepare_record function.
+            Prepare_record calls a `facebook_business` method,`ad.api_get()`, to get a ad fields. 
+            We mock this method to raise a `AttributeError` and expect the tap to retry this that function up to 5 times,
+            which is the current hard coded `max_tries` value.
+        """
+        # Mock ad object
+        mocked_ad = Mock()
+        mocked_ad.api_get = Mock()
+        mocked_ad.__getitem__ = Mock()
+        mocked_ad.api_get.side_effect = AttributeError
+
+        # # Mock get_ads function return mocked ad object
+        mocked_account = Mock()
+        mocked_account.get_ads = Mock()
+        mocked_account.get_ads.side_effect = [[mocked_ad]]
+
+        # Iterate ads object which calls prepare_record() inside and verify AttributeError is raised
+        ad_object = Ads('', mocked_account, '', '', '')
+        with self.assertRaises(AttributeError):
+            for message in ad_object:
+                pass
+
+        # verify prepare_record() function by checking call count of mocked ad.api_get()
+        self.assertEquals(mocked_ad.api_get.call_count, 5)
+
+    def test__call_get_ad_sets(self, mocked_sleep):
+        """ 
+            AdSets._call_get_ad_sets calls a `facebook_business` method,`get_ad_sets()`, to get a batch of adsets. 
+            We mock this method to raise a `AttributeError` and expect the tap to retry this that function up to 5 times,
+            which is the current hard coded `max_tries` value.
+        """
+
+        # Mock get_ad_sets function to throw AttributeError exception
+        mocked_account = Mock()
+        mocked_account.get_ad_sets = Mock()
+        mocked_account.get_ad_sets.side_effect = AttributeError
+
+        # Call _call_get_ad_sets() function of AdSets and verify AttributeError is raised
+        ad_set_object = AdSets('', mocked_account, '', '', '')
+        with self.assertRaises(AttributeError):
+            ad_set_object._call_get_ad_sets('test')
+
+        # verify get_ad_sets() is called 5 times as max 5 reties provided for function
+        self.assertEquals(mocked_account.get_ad_sets.call_count, 5)
+
+    @mock.patch("pendulum.parse")
+    def test_adset_prepare_record(self, mocked_parse, mocked_sleep):
+        """ 
+            __iter__ of AdSets calls a function _iterate which calls a nested prepare_record function.
+            Prepare_record calls a `facebook_business` method,`ad.api_get()`, to get a ad fields. 
+            We mock this method to raise a `AttributeError` and expect the tap to retry this that function up to 5 times,
+            which is the current hard coded `max_tries` value.
+        """
+
+        # Mock adset object
+        mocked_adset = Mock()
+        mocked_adset.api_get = Mock()
+        mocked_adset.__getitem__ = Mock()
+        mocked_adset.api_get.side_effect = AttributeError
+
+        # Mock get_ad_sets function return mocked ad object
+        mocked_account = Mock()
+        mocked_account.get_ad_sets = Mock()
+        mocked_account.get_ad_sets.side_effect = [[mocked_adset]]
+
+        # Iterate adset object which calls prepare_record() inside and verify AttributeError is raised
+        ad_set_object = AdSets('', mocked_account, '', '', '')
+        with self.assertRaises(AttributeError):
+            for message in ad_set_object:
+                pass
+
+        # verify prepare_record() function by checking call count of mocked ad.api_get()
+        self.assertEquals(mocked_adset.api_get.call_count, 5)
+
+    def test__call_get_campaigns(self, mocked_sleep):
+        """ 
+            Campaigns._call_get_campaigns calls a `facebook_business` method,`get_campaigns()`, to get a batch of campaigns. 
+            We mock this method to raise a `AttributeError` and expect the tap to retry this that function up to 5 times,
+            which is the current hard coded `max_tries` value.
+        """
+
+        # Mock get_campaigns function to throw AttributeError exception
+        mocked_account = Mock()
+        mocked_account.get_campaigns = Mock()
+        mocked_account.get_campaigns.side_effect = AttributeError
+
+        # Call _call_get_campaigns() function of Campaigns and verify AttributeError is raised
+        campaigns_object = Campaigns('', mocked_account, '', '', '')
+        with self.assertRaises(AttributeError):
+            campaigns_object._call_get_campaigns('test')
+
+        # verify get_campaigns() is called 5 times as max 5 reties provided for function
+        self.assertEquals(mocked_account.get_campaigns.call_count, 5)
+
+    @mock.patch("pendulum.parse")
+    def test_campaign_prepare_record(self, mocked_parse, mocked_sleep):
+        """ 
+            __iter__ of Campaigns calls a function _iterate which calls a nested prepare_record function.
+            Prepare_record calls a `facebook_business` method,`ad.api_get()`, to get a ad fields. 
+            We mock this method to raise a `AttributeError` and expect the tap to retry this that function up to 5 times,
+            which is the current hard coded `max_tries` value.
+        """
+
+        # # Mock campaign object
+        mocked_campaign = Mock()
+        mocked_campaign.api_get = Mock()
+        mocked_campaign.__getitem__ = Mock()
+        mocked_campaign.api_get.side_effect = AttributeError
+
+        # # Mock get_campaigns function return mocked ad object
+        mocked_account = Mock()
+        mocked_account.get_campaigns = Mock()
+        mocked_account.get_campaigns.side_effect = [[mocked_campaign]]
+
+        # Iterate campaigns object which calls prepare_record() inside and verify AttributeError is raised
+        campaign_object = Campaigns('', mocked_account, '', '', '')
+        with self.assertRaises(AttributeError):
+            for message in campaign_object:
+                pass
+
+        # verify prepare_record() function by checking call count of mocked ad.api_get()
+        self.assertEquals(mocked_campaign.api_get.call_count, 5)
+
+    def test_run_job(self, mocked_sleep):
+        """ 
+            AdsInsights.run_job calls a `facebook_business` method,`get_insights()`, to get a batch of insights. 
+            We mock this method to raise a `AttributeError` and expect the tap to retry this that function up to 5 times,
+            which is the current hard coded `max_tries` value.
+        """
+
+        # Mock get_insights function to throw AttributeError exception
+        mocked_account = Mock()
+        mocked_account.get_insights = Mock()
+        mocked_account.get_insights.side_effect = AttributeError
+
+        # Call run_job() function of Campaigns and verify AttributeError is raised
+        ads_insights_object = AdsInsights('', mocked_account, '', '', '', {})
+        with self.assertRaises(AttributeError):
+            ads_insights_object.run_job('test')
+
+        # verify get_insights() is called 5 times as max 5 reties provided for function
+        self.assertEquals(mocked_account.get_insights.call_count, 5)