Fix unterminated strings issue

CHANGELOG.md

@@ -1,6 +1,9 @@
 # Changelog
 
-## 1 1.2
+## 1 1.3
+  * Minor change to improve unterminated string error handling [issue #4](https://github.com/singer-io/tap-recharge/issues/4)
+
+## 1.1.2
   * Request timeout functionality added [#19](https://github.com/singer-io/tap-recharge/pull/19)
 ## 1.1.1
   * Fix bookmark access [#17](https://github.com/stitchdata/sources-utils/pull/17)

setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 
 setup(name='tap-recharge',
-      version='1.1.2',
+      version='1.1.3',
       description='Singer.io tap for extracting data from the ReCharge Payments API 2.0',
       author='jeff.huth@bytecode.io',
       classifiers=['Programming Language :: Python :: 3 :: Only'],

tap_recharge/client.py

@@ -6,7 +6,7 @@
 from requests.exceptions import Timeout
 
 LOGGER = singer.get_logger()
-REQUEST_TIMEOUT = 300
+REQUEST_TIMEOUT = 600
 
 class Server5xxError(Exception):
     pass
@@ -202,13 +202,54 @@ def request(self, method, path=None, url=None, **kwargs):
         if response.status_code != 200:
             raise_for_error(response)
 
+        # Intermittent JSONDecodeErrors when parsing JSON; Adding 2 attempts
+        # FIRST ATTEMPT
+        with metrics.http_request_timer(endpoint) as timer:
+            response = self.__session.request(method, url, stream=True, timeout=self.request_timeout, **kwargs)
+            timer.tags[metrics.Tag.http_status_code] = response.status_code
+
+        if response.status_code >= 500:
+            raise Server5xxError()
+
+        if response.status_code == 429:
+            raise Server429Error()
+
+        if response.status_code != 200:
+            raise_for_error(response)
+
+        # Catch invalid JSON (e.g. unterminated string errors)
+        try:
+            response_json = response.json()
+            return response_json, response.links
+        except ValueError as err:  # includes simplejson.decoder.JSONDecodeError
+            LOGGER.warn(err)
+
+        # SECOND ATTEMPT, if there is a ValueError (unterminated string error)
+        with metrics.http_request_timer(endpoint) as timer:
+            response = self.__session.request(
+                method,
+                url,
+                stream=True,
+                timeout=self.request_timeout,
+                **kwargs)
+            timer.tags[metrics.Tag.http_status_code] = response.status_code
+
+        if response.status_code >= 500:
+            raise Server5xxError()
+
+        if response.status_code == 429:
+            raise Server429Error()
+
+        if response.status_code != 200:
+            raise_for_error(response)
+
         # Log invalid JSON (e.g. unterminated string errors)
         try:
             response_json = response.json()
-        except Exception as err:
+            return response_json, response.links
+        except ValueError as err:  # includes simplejson.decoder.JSONDecodeError
             LOGGER.error(err)
             raise Exception(err)
-        return response_json, response.links
 
     def get(self, path, **kwargs):
         return self.request('GET', path=path, **kwargs)

tap_recharge/streams.py

@@ -14,7 +14,7 @@
 
 LOGGER = singer.get_logger()
 
-MAX_PAGE_LIMIT = 250
+MAX_PAGE_LIMIT = 50
 
 
 def get_recharge_bookmark(