From db5e45f8901a25391378527f1b890cf87834913e Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Thu, 9 Nov 2023 14:43:35 -0800 Subject: [PATCH 1/8] Enable authorization kwargs on oauth_api_connector --- parsons/utilities/oauth_api_connector.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/parsons/utilities/oauth_api_connector.py b/parsons/utilities/oauth_api_connector.py index b8356e3813..77ff1d68d2 100644 --- a/parsons/utilities/oauth_api_connector.py +++ b/parsons/utilities/oauth_api_connector.py @@ -46,6 +46,7 @@ def __init__( client_secret=None, token_url=None, auto_refresh_url=None, + authorization_kwargs=None, ): super().__init__( uri, @@ -55,16 +56,23 @@ def __init__( data_key=data_key, ) + if not authorization_kwargs: + authorization_kwargs = {} + client = BackendApplicationClient(client_id=client_id) oauth = OAuth2Session(client=client) self.token = oauth.fetch_token( - token_url=token_url, client_id=client_id, client_secret=client_secret + token_url=token_url, + client_id=client_id, + client_secret=client_secret, + **authorization_kwargs ) self.client = OAuth2Session( client_id, token=self.token, auto_refresh_url=auto_refresh_url, token_updater=self.token_saver, + auto_refresh_kwargs=authorization_kwargs, ) def request(self, url, req_type, json=None, data=None, params=None): From c86643cabe52cbe8e5b9063e99d1e0a721fd9475 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Thu, 9 Nov 2023 15:15:03 -0800 Subject: [PATCH 2/8] Expand type hints and documentation on oauth_api_connector --- parsons/utilities/oauth_api_connector.py | 44 +++++++++++------------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/parsons/utilities/oauth_api_connector.py b/parsons/utilities/oauth_api_connector.py index 77ff1d68d2..c9c9b16ae2 100644 --- a/parsons/utilities/oauth_api_connector.py +++ b/parsons/utilities/oauth_api_connector.py @@ -1,7 +1,9 @@ +import urllib.parse +from typing import Dict, Optional + from oauthlib.oauth2 import BackendApplicationClient -from requests_oauthlib import OAuth2Session from parsons.utilities.api_connector import APIConnector -import urllib.parse +from requests_oauthlib import OAuth2Session class OAuth2APIConnector(APIConnector): @@ -13,16 +15,6 @@ class OAuth2APIConnector(APIConnector): `Args:` uri: str The base uri for the api. Must include a trailing '/' (e.g. ``http://myapi.com/v1/``) - headers: dict - The request headers - auth: dict - The request authorization parameters - pagination_key: str - The name of the key in the response json where the pagination url is - located. Required for pagination. - data_key: str - The name of the key in the response json where the data is contained. Required - if the data is nested in the response json client_id: str The client id for acquiring and exchanging tokens from the OAuth2 application client_secret: str @@ -31,27 +23,33 @@ class OAuth2APIConnector(APIConnector): The URL for acquiring new tokens from the OAuth2 Application auto_refresh_url: str If provided, the URL for refreshing tokens from the OAuth2 Application + headers: dict + The request headers + pagination_key: str + The name of the key in the response json where the pagination url is + located. Required for pagination. + data_key: str + The name of the key in the response json where the data is contained. Required + if the data is nested in the response json `Returns`: OAuthAPIConnector class """ def __init__( self, - uri, - headers=None, - auth=None, - pagination_key=None, - data_key=None, - client_id=None, - client_secret=None, - token_url=None, - auto_refresh_url=None, - authorization_kwargs=None, + uri: str, + client_id: str, + client_secret: str, + token_url: str, + auto_refresh_url: Optional[str], + headers: Optional[Dict[str, str]] = None, + pagination_key: Optional[str] = None, + data_key: Optional[str] = None, + authorization_kwargs: Optional[Dict[str, str]] = None, ): super().__init__( uri, headers=headers, - auth=auth, pagination_key=pagination_key, data_key=data_key, ) From 3657b29d921e4296a0f5cc6216f830e8298f4e5b Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Thu, 9 Nov 2023 14:48:28 -0800 Subject: [PATCH 3/8] Use isinstance for type check to satisfy flake8 --- parsons/zoom/zoom.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/parsons/zoom/zoom.py b/parsons/zoom/zoom.py index 7526d01922..eba7618eef 100644 --- a/parsons/zoom/zoom.py +++ b/parsons/zoom/zoom.py @@ -353,7 +353,7 @@ def get_meeting_poll_metadata(self, meeting_id, poll_id) -> Table: endpoint = f"meetings/{meeting_id}/polls/{poll_id}" tbl = self._get_request(endpoint=endpoint, data_key="questions") - if type(tbl) == dict: + if isinstance(tbl, dict): logger.debug(f"No poll data returned for poll ID {poll_id}") return Table(tbl) @@ -380,7 +380,7 @@ def get_meeting_all_polls_metadata(self, meeting_id) -> Table: endpoint = f"meetings/{meeting_id}/polls" tbl = self._get_request(endpoint=endpoint, data_key="polls") - if type(tbl) == dict: + if isinstance(tbl, dict): logger.debug(f"No poll data returned for meeting ID {meeting_id}") return Table(tbl) @@ -405,7 +405,7 @@ def get_past_meeting_poll_metadata(self, meeting_id) -> Table: endpoint = f"past_meetings/{meeting_id}/polls" tbl = self._get_request(endpoint=endpoint, data_key="questions") - if type(tbl) == dict: + if isinstance(tbl, dict): logger.debug(f"No poll data returned for meeting ID {meeting_id}") return Table(tbl) @@ -432,7 +432,7 @@ def get_webinar_poll_metadata(self, webinar_id, poll_id) -> Table: endpoint = f"webinars/{webinar_id}/polls/{poll_id}" tbl = self._get_request(endpoint=endpoint, data_key="questions") - if type(tbl) == dict: + if isinstance(tbl, dict): logger.debug(f"No poll data returned for poll ID {poll_id}") return Table(tbl) @@ -459,7 +459,7 @@ def get_webinar_all_polls_metadata(self, webinar_id) -> Table: endpoint = f"webinars/{webinar_id}/polls" tbl = self._get_request(endpoint=endpoint, data_key="polls") - if type(tbl) == dict: + if isinstance(tbl, dict): logger.debug(f"No poll data returned for webinar ID {webinar_id}") return Table(tbl) @@ -484,7 +484,7 @@ def get_past_webinar_poll_metadata(self, webinar_id) -> Table: endpoint = f"past_webinars/{webinar_id}/polls" tbl = self._get_request(endpoint=endpoint, data_key="questions") - if type(tbl) == dict: + if isinstance(tbl, dict): logger.debug(f"No poll data returned for webinar ID {webinar_id}") return Table(tbl) @@ -502,7 +502,7 @@ def get_meeting_poll_results(self, meeting_id) -> Table: endpoint = f"report/meetings/{meeting_id}/polls" tbl = self._get_request(endpoint=endpoint, data_key="questions") - if type(tbl) == dict: + if isinstance(tbl, dict): logger.debug(f"No poll data returned for meeting ID {meeting_id}") return Table(tbl) @@ -520,7 +520,7 @@ def get_webinar_poll_results(self, webinar_id) -> Table: endpoint = f"report/webinars/{webinar_id}/polls" tbl = self._get_request(endpoint=endpoint, data_key="questions") - if type(tbl) == dict: + if isinstance(tbl, dict): logger.debug(f"No poll data returned for webinar ID {webinar_id}") return Table(tbl) From 39ad845d503626cd26cf22492ba6fb8effff35ea Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Thu, 9 Nov 2023 14:49:01 -0800 Subject: [PATCH 4/8] Use oauth_api_connector in Zoom connector --- parsons/zoom/zoom.py | 58 ++++++-------------------------------------- 1 file changed, 8 insertions(+), 50 deletions(-) diff --git a/parsons/zoom/zoom.py b/parsons/zoom/zoom.py index eba7618eef..dbcb45d12b 100644 --- a/parsons/zoom/zoom.py +++ b/parsons/zoom/zoom.py @@ -1,9 +1,7 @@ from parsons.utilities import check_env -from parsons.utilities.api_connector import APIConnector +from parsons.utilities.oauth_api_connector import OAuth2APIConnector from parsons import Table import logging -import jwt -import datetime import uuid logger = logging.getLogger(__name__) @@ -32,55 +30,15 @@ def __init__(self, account_id=None, client_id=None, client_secret=None): self.client_id = check_env.check("ZOOM_CLIENT_ID", client_id) self.__client_secret = check_env.check("ZOOM_CLIENT_SECRET", client_secret) - self.client = APIConnector(uri=ZOOM_URI) - - access_token = self.__generate_access_token() - - self.client.headers = { - "Authorization": f"Bearer {access_token}", - "Content-type": "application/json", - } - - def __generate_access_token(self) -> str: - """ - Uses Zoom's OAuth callback URL to generate an access token to query the Zoom API - - `Returns`: - String representation of access token - """ - - temp_client = APIConnector( - uri=ZOOM_URI, auth=(self.client_id, self.__client_secret) + self.client = OAuth2APIConnector( + uri=ZOOM_URI, + client_id=self.client_id, + client_secret=self.__client_secret, + token_url=ZOOM_AUTH_CALLBACK, + auto_refresh_url=ZOOM_AUTH_CALLBACK, + authorization_kwargs={"account_id": self.account_id}, ) - resp = temp_client.post_request( - ZOOM_AUTH_CALLBACK, - data={ - "grant_type": "account_credentials", - "account_id": self.account_id, - }, - ) - - return resp["access_token"] - - def __refresh_header_token(self): - """ - NOTE: This function is deprecated as Zoom's API moves to an OAuth strategy on 9/1 - - Generate a token that is valid for 30 seconds and update header. Full documentation - on JWT generation using Zoom API: https://marketplace.zoom.us/docs/guides/auth/jwt - """ - - payload = { - "iss": self.api_key, - "exp": int(datetime.datetime.now().timestamp() + 30), - } - token = jwt.encode(payload, self.api_secret, algorithm="HS256") - self.client.headers = { - "authorization": f"Bearer {token}", - "content-type": "application/json", - } - def _get_request(self, endpoint, data_key, params=None, **kwargs): """ TODO: Consider increasing default page size. From 857d179436a044bcd08abe77ba9b5efd8aebee25 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Thu, 9 Nov 2023 14:44:50 -0800 Subject: [PATCH 5/8] Use oauth_api_connector in Catalist connector --- parsons/catalist/catalist.py | 46 ++++++++++++------------------------ 1 file changed, 15 insertions(+), 31 deletions(-) diff --git a/parsons/catalist/catalist.py b/parsons/catalist/catalist.py index 4779f70648..b090c91793 100644 --- a/parsons/catalist/catalist.py +++ b/parsons/catalist/catalist.py @@ -12,10 +12,9 @@ from typing import Optional, Union, Dict, List from zipfile import ZipFile -import requests from parsons.etl import Table from parsons.sftp import SFTP -from parsons.utilities.api_connector import APIConnector +from parsons.utilities.oauth_api_connector import OAuth2APIConnector logger = logging.getLogger(__name__) @@ -62,36 +61,19 @@ def __init__( client_secret: str, sftp_username: str, sftp_password: str, + client_audience: Optional[str] = None, ) -> None: self.client_id = client_id self.client_secret = client_secret - self.fetch_token() - self.connection = APIConnector("http://api.catalist.us/mapi/") - self.sftp = SFTP("t.catalist.us", sftp_username, sftp_password) - - @property - def token(self) -> str: - """If token is not yet fetched or has expired, fetch new token.""" - if not (self._token and time.time() < self._token_expired_at): - self.fetch_token() - return self._token - - def fetch_token(self) -> None: - """Fetch auth0 token to be used with Catalist API.""" - url = "https://auth.catalist.us/oauth/token" - payload = { - "grant_type": "client_credentials", - "audience": "catalist_api_m_prod", - } - response = requests.post( - url, json=payload, auth=(self.client_id, self.client_secret) + self.connection = OAuth2APIConnector( + "https://api.catalist.us/mapi/", + client_id=client_id, + client_secret=client_secret, + authorization_kwargs={"audience": client_audience or "catalist_api_m_prod"}, + token_url="https://auth.catalist.us/oauth/token", + auto_refresh_url="https://auth.catalist.us/oauth/token", ) - data = response.json() - - self._token = data["access_token"] - self._token_expired_at = time.time() + data["expires_in"] - - logger.info("Token refreshed.") + self.sftp = SFTP("t.catalist.us", sftp_username, sftp_password) def load_table_to_sftp( self, table: Table, input_subfolder: Optional[str] = None @@ -241,7 +223,9 @@ def upload( endpoint = "/".join(endpoint_params) # Assemble query parameters - query_params: Dict[str, Union[str, int]] = {"token": self.token} + query_params: Dict[str, Union[str, int]] = { + "token": self.connection.token["access_token"] + } if copy_to_sandbox: query_params["copyToSandbox"] = "true" if static_values: @@ -308,7 +292,7 @@ def action( logger.debug(f"Executing request to endpoint {self.connection.uri + endpoint}") - query_params = {"token": self.token} + query_params = {"token": self.connection.token["access_token"]} if copy_to_sandbox: query_params["copyToSandbox"] = "true" if export_filename_suffix: @@ -323,7 +307,7 @@ def action( def status(self, id: str) -> dict: """Check status of a match job.""" endpoint = "/".join(["status", "id", id]) - query_params = {"token": self.token} + query_params = {"token": self.connection.token["access_token"]} result = self.connection.get_request(endpoint, params=query_params) return result From 89a37ecfc1fb53981e2908c96808873156d84177 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Fri, 9 Feb 2024 16:02:35 -0500 Subject: [PATCH 6/8] Enable configurable grant_type for oauth connector --- parsons/utilities/oauth_api_connector.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parsons/utilities/oauth_api_connector.py b/parsons/utilities/oauth_api_connector.py index c9c9b16ae2..fa25e018be 100644 --- a/parsons/utilities/oauth_api_connector.py +++ b/parsons/utilities/oauth_api_connector.py @@ -45,6 +45,7 @@ def __init__( headers: Optional[Dict[str, str]] = None, pagination_key: Optional[str] = None, data_key: Optional[str] = None, + grant_type: str = "client_credentials", authorization_kwargs: Optional[Dict[str, str]] = None, ): super().__init__( @@ -58,6 +59,7 @@ def __init__( authorization_kwargs = {} client = BackendApplicationClient(client_id=client_id) + client.grant_type = grant_type oauth = OAuth2Session(client=client) self.token = oauth.fetch_token( token_url=token_url, From f984459e0a5fd8c89cb83c01ef9030cbc140ab59 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Fri, 9 Feb 2024 16:02:48 -0500 Subject: [PATCH 7/8] Set grant_type to "account_credentials" for Zoom oauth connector --- parsons/zoom/zoom.py | 1 + 1 file changed, 1 insertion(+) diff --git a/parsons/zoom/zoom.py b/parsons/zoom/zoom.py index dbcb45d12b..b71b354566 100644 --- a/parsons/zoom/zoom.py +++ b/parsons/zoom/zoom.py @@ -36,6 +36,7 @@ def __init__(self, account_id=None, client_id=None, client_secret=None): client_secret=self.__client_secret, token_url=ZOOM_AUTH_CALLBACK, auto_refresh_url=ZOOM_AUTH_CALLBACK, + grant_type="account_credentials", authorization_kwargs={"account_id": self.account_id}, ) From 3960dbb3f3121a5be434a4b6e7c69ed12d715063 Mon Sep 17 00:00:00 2001 From: Austin Weisgrau Date: Mon, 19 Feb 2024 12:02:55 -0800 Subject: [PATCH 8/8] Ensure controlshift hostname starts with https:// This is required for oauth2 --- parsons/controlshift/controlshift.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/parsons/controlshift/controlshift.py b/parsons/controlshift/controlshift.py index 51608c6770..5c9d6d06fa 100644 --- a/parsons/controlshift/controlshift.py +++ b/parsons/controlshift/controlshift.py @@ -12,7 +12,7 @@ class Controlshift(object): `Args:` hostname: str The URL for the homepage/login page of the organization's Controlshift - instance (e.g. demo.controlshift.app). Not required if + instance (e.g. https://demo.controlshift.app). Not required if ``CONTROLSHIFT_HOSTNAME`` env variable is set. client_id: str The Client ID for your REST API Application. Not required if @@ -27,6 +27,13 @@ class Controlshift(object): def __init__(self, hostname=None, client_id=None, client_secret=None): self.hostname = check_env.check("CONTROLSHIFT_HOSTNAME", hostname) + + # Hostname must start with 'https://' + if self.hostname.startswith("http://"): + self.hostname = self.hostname.replace("http://", "https://") + if not self.hostname.startswith("https://"): + self.hostname = "https://" + self.hostname + token_url = f"{self.hostname}/oauth/token" self.client = OAuth2APIConnector( self.hostname,