From 3fbd7dd1ff92b9389a83995036237241acdb8825 Mon Sep 17 00:00:00 2001 From: Dan Mosora <30501696+dmosorast@users.noreply.github.com> Date: Fri, 31 Dec 2021 15:15:52 +0000 Subject: [PATCH 1/4] Request all types of subscriptions --- tap_stripe/__init__.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/tap_stripe/__init__.py b/tap_stripe/__init__.py index 47f83d30..ab03a067 100755 --- a/tap_stripe/__init__.py +++ b/tap_stripe/__init__.py @@ -28,7 +28,7 @@ 'key_properties': ['id', 'invoice']}, 'transfers': {'sdk_object': stripe.Transfer, 'key_properties': ['id']}, 'coupons': {'sdk_object': stripe.Coupon, 'key_properties': ['id']}, - 'subscriptions': {'sdk_object': stripe.Subscription, 'key_properties': ['id']}, + 'subscriptions': {'sdk_object': stripe.Subscription, 'key_properties': ['id'], 'params': {'status': 'all'}}, 'subscription_items': {'sdk_object': stripe.SubscriptionItem, 'key_properties': ['id']}, 'balance_transactions': {'sdk_object': stripe.BalanceTransaction, 'key_properties': ['id']}, @@ -371,14 +371,15 @@ def reduce_foreign_keys(rec, stream_name): return rec -def paginate(sdk_obj, filter_key, start_date, end_date, limit=100): +def paginate(sdk_obj, filter_key, start_date, end_date, limit=100, **kwargs): yield from sdk_obj.list( limit=limit, stripe_account=Context.config.get('account_id'), # None passed to starting_after appears to retrieve # all of them so this should always be safe. **{filter_key + "[gte]": start_date, - filter_key + "[lt]": end_date} + filter_key + "[lt]": end_date}, + **kwargs ).auto_paging_iter() @@ -444,6 +445,7 @@ def sync_stream(stream_name): # immutable streams at first to confirm the suspicion. start_window -= IMMUTABLE_STREAM_LOOKBACK + custom_params = STREAM_SDK_OBJECTS[stream_name].get('params', {}) # NB: We observed records coming through newest->oldest and so # date-windowing was added and the tap only bookmarks after it has # gotten through a date window @@ -454,8 +456,7 @@ def sync_stream(stream_name): stop_window = end_time for stream_obj in paginate(STREAM_SDK_OBJECTS[stream_name]['sdk_object'], - filter_key, start_window, stop_window): - + filter_key, start_window, stop_window, **custom_params): # get the replication key value from the object rec = unwrap_data_objects(stream_obj.to_dict_recursive()) rec = reduce_foreign_keys(rec, stream_name) From a536536c058418fb88d3eafdc33e64faeb6fd9f6 Mon Sep 17 00:00:00 2001 From: Dan Mosora <30501696+dmosorast@users.noreply.github.com> Date: Fri, 31 Dec 2021 15:20:14 +0000 Subject: [PATCH 2/4] 1.5.1 and changelog --- CHANGELOG.md | 25 ++++++++++++++----------- setup.py | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5bcb56a..050b5d5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,37 +1,40 @@ # Changelog -** 1.5.0 +## 1.5.1 + * Subscriptions stream will now request subscriptions of all statuses [#113](https://github.com/singer-io/tap-stripe/pull/113) + +## 1.5.0 * Add schema for card_present charges [#101](https://github.com/singer-io/tap-stripe/pull/101) -** 1.4.9 +## 1.4.9 * Allow partial days in the `date_window_size` config value [#100](https://github.com/singer-io/tap-stripe/pull/100) -** 1.4.8 +## 1.4.8 * Reverts 1.4.7 [#82](https://github.com/singer-io/tap-stripe/pull/82) -** 1.4.7 +## 1.4.7 * Updates singer-python from 5.5.1 to 5.12.1 [#81](https://github.com/singer-io/tap-stripe/pull/81) -** 1.4.6 +## 1.4.6 * Removed fields that caused transform errors -** 1.4.5 +## 1.4.5 * Added various schema [#77](https://github.com/singer-io/tap-stripe/pull/77) -** 1.4.4 +## 1.4.4 * Use the default timeout in the stripe client [#61](https://github.com/singer-io/tap-stripe/pull/61) -** 1.4.3 +## 1.4.3 * Revert 1.4.2 changes from #59 [#60](https://github.com/singer-io/tap-stripe/pull/60) * Remove invalid and unused schema pieces [#60](https://github.com/singer-io/tap-stripe/pull/60) -** 1.4.2 +## 1.4.2 * Revert 1.4.1 [#59](https://github.com/singer-io/tap-stripe/pull/59) -** 1.4.1 +## 1.4.1 * Add functionality that recursively converts `StripeObject`s to dictionaries. [#53](https://github.com/singer-io/tap-stripe/pull/53) -** 1.4.0 +## 1.4.0 * Added the `payment_method_details` field to the charges stream. [#49](https://github.com/singer-io/tap-stripe/pull/49) ## 1.3.7 diff --git a/setup.py b/setup.py index 5fa0ad6f..b980b782 100755 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name="tap-stripe", - version="1.5.0", + version="1.5.1", description="Singer.io tap for extracting data", author="Stitch", url="http://singer.io", From 4ae90a8e05bf31a4ccb14dca9d77f1ae3a01bf31 Mon Sep 17 00:00:00 2001 From: Dan Mosora <30501696+dmosorast@users.noreply.github.com> Date: Fri, 31 Dec 2021 15:29:55 +0000 Subject: [PATCH 3/4] Revert "Request all types of subscriptions" in favor of PR #107 This reverts commit 3fbd7dd1ff92b9389a83995036237241acdb8825. --- tap_stripe/__init__.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tap_stripe/__init__.py b/tap_stripe/__init__.py index ab03a067..47f83d30 100755 --- a/tap_stripe/__init__.py +++ b/tap_stripe/__init__.py @@ -28,7 +28,7 @@ 'key_properties': ['id', 'invoice']}, 'transfers': {'sdk_object': stripe.Transfer, 'key_properties': ['id']}, 'coupons': {'sdk_object': stripe.Coupon, 'key_properties': ['id']}, - 'subscriptions': {'sdk_object': stripe.Subscription, 'key_properties': ['id'], 'params': {'status': 'all'}}, + 'subscriptions': {'sdk_object': stripe.Subscription, 'key_properties': ['id']}, 'subscription_items': {'sdk_object': stripe.SubscriptionItem, 'key_properties': ['id']}, 'balance_transactions': {'sdk_object': stripe.BalanceTransaction, 'key_properties': ['id']}, @@ -371,15 +371,14 @@ def reduce_foreign_keys(rec, stream_name): return rec -def paginate(sdk_obj, filter_key, start_date, end_date, limit=100, **kwargs): +def paginate(sdk_obj, filter_key, start_date, end_date, limit=100): yield from sdk_obj.list( limit=limit, stripe_account=Context.config.get('account_id'), # None passed to starting_after appears to retrieve # all of them so this should always be safe. **{filter_key + "[gte]": start_date, - filter_key + "[lt]": end_date}, - **kwargs + filter_key + "[lt]": end_date} ).auto_paging_iter() @@ -445,7 +444,6 @@ def sync_stream(stream_name): # immutable streams at first to confirm the suspicion. start_window -= IMMUTABLE_STREAM_LOOKBACK - custom_params = STREAM_SDK_OBJECTS[stream_name].get('params', {}) # NB: We observed records coming through newest->oldest and so # date-windowing was added and the tap only bookmarks after it has # gotten through a date window @@ -456,7 +454,8 @@ def sync_stream(stream_name): stop_window = end_time for stream_obj in paginate(STREAM_SDK_OBJECTS[stream_name]['sdk_object'], - filter_key, start_window, stop_window, **custom_params): + filter_key, start_window, stop_window): + # get the replication key value from the object rec = unwrap_data_objects(stream_obj.to_dict_recursive()) rec = reduce_foreign_keys(rec, stream_name) From cb611331a16d03e1ed8228f24881cb81ed42e901 Mon Sep 17 00:00:00 2001 From: Somtom Date: Wed, 22 Dec 2021 15:06:31 +0100 Subject: [PATCH 4/4] Sync all stripe subscriptions not only the active ones --- tap_stripe/__init__.py | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/tap_stripe/__init__.py b/tap_stripe/__init__.py index 47f83d30..cf082644 100755 --- a/tap_stripe/__init__.py +++ b/tap_stripe/__init__.py @@ -28,7 +28,11 @@ 'key_properties': ['id', 'invoice']}, 'transfers': {'sdk_object': stripe.Transfer, 'key_properties': ['id']}, 'coupons': {'sdk_object': stripe.Coupon, 'key_properties': ['id']}, - 'subscriptions': {'sdk_object': stripe.Subscription, 'key_properties': ['id']}, + 'subscriptions': { + 'sdk_object': stripe.Subscription, + 'key_properties': ['id'], + 'request_args': {'status': 'all'} + }, 'subscription_items': {'sdk_object': stripe.SubscriptionItem, 'key_properties': ['id']}, 'balance_transactions': {'sdk_object': stripe.BalanceTransaction, 'key_properties': ['id']}, @@ -371,14 +375,15 @@ def reduce_foreign_keys(rec, stream_name): return rec -def paginate(sdk_obj, filter_key, start_date, end_date, limit=100): +def paginate(sdk_obj, filter_key, start_date, end_date, request_args=None, limit=100): yield from sdk_obj.list( limit=limit, stripe_account=Context.config.get('account_id'), # None passed to starting_after appears to retrieve # all of them so this should always be safe. **{filter_key + "[gte]": start_date, - filter_key + "[lt]": end_date} + filter_key + "[lt]": end_date}, + **request_args or {} ).auto_paging_iter() @@ -453,8 +458,13 @@ def sync_stream(stream_name): if stop_window > end_time: stop_window = end_time - for stream_obj in paginate(STREAM_SDK_OBJECTS[stream_name]['sdk_object'], - filter_key, start_window, stop_window): + for stream_obj in paginate( + STREAM_SDK_OBJECTS[stream_name]['sdk_object'], + filter_key, + start_window, + stop_window, + STREAM_SDK_OBJECTS[stream_name].get('request_args') + ): # get the replication key value from the object rec = unwrap_data_objects(stream_obj.to_dict_recursive())