From 1f6143290275fae5efd3edd24fee25d92c62d098 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 18 Apr 2023 18:41:07 -0600 Subject: [PATCH 1/5] feat: Explicitly support URL params in string form --- singer_sdk/streams/rest.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index fe83bea36..4389fecae 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -279,18 +279,33 @@ def get_url_params( self, context: dict | None, # noqa: ARG002 next_page_token: _TToken | None, # noqa: ARG002 - ) -> dict[str, Any]: + ) -> dict[str, Any] | str: """Return a dictionary of values to be used in URL parameterization. If paging is supported, developers may override with specific paging logic. + If you source needs special handling so, for example, parenthesis are not + encoded, you can return a string constructed with `urllib.parse.urlencode`_: + + .. code-block:: python + from urllib.parse import urlencode + + class MyStream(RESTStream): + def get_url_params(self, context, next_page_token): + params = {"key": "(a,b,c)"} + return urlencode(params, safe="()") + Args: context: Stream partition or context dictionary. next_page_token: Token, page number or any request argument to request the next page of data. Returns: - Dictionary of URL query parameters to use in the request. + Dictionary or encoded string with URL query parameters to use in the + request. + + .. _urllib.parse.urlencode: + https://docs.python.org/3/library/urllib.parse.html#urllib.parse.urlencode """ return {} @@ -341,7 +356,7 @@ def prepare_request( """ http_method = self.rest_method url: str = self.get_url(context) - params: dict = self.get_url_params(context, next_page_token) + params: dict | str = self.get_url_params(context, next_page_token) request_data = self.prepare_request_payload(context, next_page_token) headers = self.http_headers From 03527e88a978af152bf13c67dc0ca21348447039 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Edgar=20Rami=CC=81rez=20Mondrago=CC=81n?= Date: Tue, 18 Apr 2023 18:54:08 -0600 Subject: [PATCH 2/5] Add missing line break --- singer_sdk/streams/rest.py | 1 + 1 file changed, 1 insertion(+) diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index 4389fecae..60e7b0bb7 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -288,6 +288,7 @@ def get_url_params( encoded, you can return a string constructed with `urllib.parse.urlencode`_: .. code-block:: python + from urllib.parse import urlencode class MyStream(RESTStream): From a58ec53578600b22ab1367f86ec7c08dd5c6ba85 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 18 Apr 2023 20:57:57 -0600 Subject: [PATCH 3/5] Update singer_sdk/streams/rest.py --- singer_sdk/streams/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index 60e7b0bb7..644650e6a 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -280,7 +280,7 @@ def get_url_params( context: dict | None, # noqa: ARG002 next_page_token: _TToken | None, # noqa: ARG002 ) -> dict[str, Any] | str: - """Return a dictionary of values to be used in URL parameterization. + """Return a dictionary or string of URL query parameters. If paging is supported, developers may override with specific paging logic. From 7730bfbf374433cabb5981ec61bfdd67b133d51a Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 18 Apr 2023 20:58:02 -0600 Subject: [PATCH 4/5] Update singer_sdk/streams/rest.py --- singer_sdk/streams/rest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index 644650e6a..a19cd5044 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -284,7 +284,7 @@ def get_url_params( If paging is supported, developers may override with specific paging logic. - If you source needs special handling so, for example, parenthesis are not + If your source needs special handling and, for example, parentheses should not be encoded, you can return a string constructed with `urllib.parse.urlencode`_: .. code-block:: python From 3634b4c02cc9c82283be3112ad2b58f1b66bdad6 Mon Sep 17 00:00:00 2001 From: "Edgar R. M" Date: Tue, 18 Apr 2023 21:01:01 -0600 Subject: [PATCH 5/5] Update singer_sdk/streams/rest.py --- singer_sdk/streams/rest.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/singer_sdk/streams/rest.py b/singer_sdk/streams/rest.py index a19cd5044..da4c9a540 100644 --- a/singer_sdk/streams/rest.py +++ b/singer_sdk/streams/rest.py @@ -284,8 +284,9 @@ def get_url_params( If paging is supported, developers may override with specific paging logic. - If your source needs special handling and, for example, parentheses should not be - encoded, you can return a string constructed with `urllib.parse.urlencode`_: + If your source needs special handling and, for example, parentheses should not + be encoded, you can return a string constructed with + `urllib.parse.urlencode`_: .. code-block:: python