From 49468f63a27a2a6a02a9aa0fa71d7c36df09cb84 Mon Sep 17 00:00:00 2001 From: Corma Martinez del Rio <66973815+cmdelrio@users.noreply.github.com> Date: Thu, 11 Aug 2022 10:21:15 -0600 Subject: [PATCH 1/6] mobilecommons class --- parsons/mobilecommons/mobilecommons.py | 32 ++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 parsons/mobilecommons/mobilecommons.py diff --git a/parsons/mobilecommons/mobilecommons.py b/parsons/mobilecommons/mobilecommons.py new file mode 100644 index 0000000000..274a70a395 --- /dev/null +++ b/parsons/mobilecommons/mobilecommons.py @@ -0,0 +1,32 @@ +from parsons.utilities import check_env +from parsons.utilities.api_connector import APIConnector +from parsons import Table +import logging + +logger = logging.getLogger(__name__) + +MC_URI = 'https://secure.mcommons.com/api/' + +class MobileCommons: + """ + Instantiate the MobileCommons class. + + `Args:` + username: str + A valid email address connected toa MobileCommons accouont. Not required if + ``MOBILE_COMMONS_USERNAME`` env variable is set. + password: str + Password associated with zoom account. Not required if ``MOBILE_COMMONS_PASSWORD`` + env variable set. + companyid: str + The company id of the MobileCommons organization to connect to. Not required if + username and password are for an account associated with only one MobileCommons + organization. + """ + def __init__(self, username=None, password=None, companyid=None): + self.username = check_env.check('MOBILE_COMMONS_USERNAME', username) + self.password = check_env.check('MOBILE_COMMONS_USERNAME', password) + self.companyid = companyid + self.client = APIConnector(uri=MC_URI, auth=(self.username,self.password)) + + From bcaafa5eaf6f66290c2996b570d8035cc52e41e7 Mon Sep 17 00:00:00 2001 From: Corma Martinez del Rio <66973815+cmdelrio@users.noreply.github.com> Date: Thu, 11 Aug 2022 11:51:57 -0600 Subject: [PATCH 2/6] Update __init__.py --- parsons/mobilecommons/__init__.py | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 parsons/mobilecommons/__init__.py diff --git a/parsons/mobilecommons/__init__.py b/parsons/mobilecommons/__init__.py new file mode 100644 index 0000000000..d25b28511f --- /dev/null +++ b/parsons/mobilecommons/__init__.py @@ -0,0 +1,5 @@ +from parsons.mobilecommons.mobilecommons import MobileCommons + +__all__ = [ + 'MobileCommons' +] From 74337717e22c80a9e0cdcb76d0735e792686862e Mon Sep 17 00:00:00 2001 From: Corma Martinez del Rio <66973815+cmdelrio@users.noreply.github.com> Date: Thu, 11 Aug 2022 14:51:07 -0600 Subject: [PATCH 3/6] get broadcasts --- parsons/mobilecommons/mobilecommons.py | 42 +++++++++++++++++++++++--- requirements.txt | 1 + 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/parsons/mobilecommons/mobilecommons.py b/parsons/mobilecommons/mobilecommons.py index 274a70a395..e0cf84ea2e 100644 --- a/parsons/mobilecommons/mobilecommons.py +++ b/parsons/mobilecommons/mobilecommons.py @@ -1,6 +1,9 @@ from parsons.utilities import check_env from parsons.utilities.api_connector import APIConnector from parsons import Table +from bs4 import BeautifulSoup +from requests import HTTPError +import xmltodict import logging logger = logging.getLogger(__name__) @@ -14,9 +17,9 @@ class MobileCommons: `Args:` username: str A valid email address connected toa MobileCommons accouont. Not required if - ``MOBILE_COMMONS_USERNAME`` env variable is set. + ``MOBILECOMMONS_USERNAME`` env variable is set. password: str - Password associated with zoom account. Not required if ``MOBILE_COMMONS_PASSWORD`` + Password associated with zoom account. Not required if ``MOBILECOMMONS_PASSWORD`` env variable set. companyid: str The company id of the MobileCommons organization to connect to. Not required if @@ -24,9 +27,38 @@ class MobileCommons: organization. """ def __init__(self, username=None, password=None, companyid=None): - self.username = check_env.check('MOBILE_COMMONS_USERNAME', username) - self.password = check_env.check('MOBILE_COMMONS_USERNAME', password) - self.companyid = companyid + self.username = check_env.check('MOBILECOMMONS_USERNAME', username) + self.password = check_env.check('MOBILECOMMONS_PASSWORD', password) + self.companyid_param = f'?company={companyid}' if companyid else '' self.client = APIConnector(uri=MC_URI, auth=(self.username,self.password)) + def get_broadcasts(self, start_time=None, end_time=None, status=None, campaign_id=None, + limit=None): + broadcast_table = Table() + + #MC page limit is 1000 for broadcasts + page_limit = 1000 if limit > 1000 or limit is None else limit + params = f'limit={str(page_limit)}' + response = self.client.request('broadcasts' + self.companyid_param, 'GET', params=params) + if response.status_code == 200: + response_dict = xmltodict.parse(response.text) + response_table = Table(response_dict['response']['broadcasts']['broadcast']) + broadcast_table.concat(response_table) + page_count = response_dict['response']['broadcasts']['page_count'] + if page_count > 1: + for i in page_count: + page_params = params + f'&page={str(i)}' + response = self.client.request('broadcasts' + self.companyid_param, 'GET', + params=page_params) + response_dict = xmltodict.parse(response.text) + response_table = Table(response_dict['response']['broadcasts']['broadcast']) + broadcast_table.concat(response_table) + else: + error = f'Response Code {str(response.status_code)}' + error_html = BeautifulSoup(response.text, features='html.parser') + error += '\n' + error_html.h4.next + error += '\n' + error_html.p.next + raise HTTPError(error) + + return broadcast_table diff --git a/requirements.txt b/requirements.txt index fa991c0264..c8bbdda9a9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,7 @@ surveygizmo==1.2.3 PyJWT==2.0.1 # Otherwise `import jwt` would refer to python-jwt package SQLAlchemy==1.3.23 requests_oauthlib==1.3.0 +bs4==0.0.1 # Testing Requirements requests-mock==1.5.2 From 8ec4625887b34bc16d4f7d58c56e58f39ffa6fdf Mon Sep 17 00:00:00 2001 From: Corma Martinez del Rio <66973815+cmdelrio@users.noreply.github.com> Date: Tue, 16 Aug 2022 17:26:52 -0600 Subject: [PATCH 4/6] fix get broadcast request --- parsons/mobilecommons/mobilecommons.py | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/parsons/mobilecommons/mobilecommons.py b/parsons/mobilecommons/mobilecommons.py index e0cf84ea2e..76e1e73c0a 100644 --- a/parsons/mobilecommons/mobilecommons.py +++ b/parsons/mobilecommons/mobilecommons.py @@ -38,22 +38,26 @@ def get_broadcasts(self, start_time=None, end_time=None, status=None, campaign_i broadcast_table = Table() #MC page limit is 1000 for broadcasts - page_limit = 1000 if limit > 1000 or limit is None else limit + page_limit = 20 if limit > 20 or limit is None else limit params = f'limit={str(page_limit)}' response = self.client.request('broadcasts' + self.companyid_param, 'GET', params=params) if response.status_code == 200: - response_dict = xmltodict.parse(response.text) + response_dict = xmltodict.parse(response.text, attr_prefix='', cdata_key='', + dict_constructor=dict) response_table = Table(response_dict['response']['broadcasts']['broadcast']) + response_table.unpack_dict('campaign') broadcast_table.concat(response_table) - page_count = response_dict['response']['broadcasts']['page_count'] - if page_count > 1: - for i in page_count: - page_params = params + f'&page={str(i)}' - response = self.client.request('broadcasts' + self.companyid_param, 'GET', - params=page_params) - response_dict = xmltodict.parse(response.text) - response_table = Table(response_dict['response']['broadcasts']['broadcast']) - broadcast_table.concat(response_table) + page_count = int(response_dict['response']['broadcasts']['page_count']) + i = 2 + while i <= page_count: + page_params = params + f'&page={str(i)}' + response = self.client.request('broadcasts' + self.companyid_param, 'GET', + params=page_params) + response_dict = xmltodict.parse(response.text, attr_prefix='', cdata_key='', + dict_constructor=dict) + response_table = Table(response_dict['response']['broadcasts']['broadcast']) + broadcast_table.concat(response_table) + i += 1 else: error = f'Response Code {str(response.status_code)}' error_html = BeautifulSoup(response.text, features='html.parser') From e48f628e32ab05de1448d0a32ef765d7537c9eae Mon Sep 17 00:00:00 2001 From: Corma Martinez del Rio <66973815+cmdelrio@users.noreply.github.com> Date: Tue, 16 Aug 2022 18:53:15 -0600 Subject: [PATCH 5/6] Add mc_get_request method --- parsons/mobilecommons/mobilecommons.py | 46 ++++++++++++++++---------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/parsons/mobilecommons/mobilecommons.py b/parsons/mobilecommons/mobilecommons.py index 76e1e73c0a..9d294cfb8d 100644 --- a/parsons/mobilecommons/mobilecommons.py +++ b/parsons/mobilecommons/mobilecommons.py @@ -5,6 +5,7 @@ from requests import HTTPError import xmltodict import logging +import math logger = logging.getLogger(__name__) @@ -32,31 +33,34 @@ def __init__(self, username=None, password=None, companyid=None): self.companyid_param = f'?company={companyid}' if companyid else '' self.client = APIConnector(uri=MC_URI, auth=(self.username,self.password)) - def get_broadcasts(self, start_time=None, end_time=None, status=None, campaign_id=None, - limit=None): - - broadcast_table = Table() - - #MC page limit is 1000 for broadcasts - page_limit = 20 if limit > 20 or limit is None else limit - params = f'limit={str(page_limit)}' - response = self.client.request('broadcasts' + self.companyid_param, 'GET', params=params) + def mc_get_request(self, endpoint, data_key, params, cols_to_unpack, limit): + final_table = Table() + page_limit = 1000 if limit > 1000 or limit is None else limit + logger.info(f'Working on fetching first {page_limit} rows. ' + f'Each 1000 rows will take aproximately 40 seconds to fetch.') + params += f'limit={str(page_limit)}' + response = self.client.request(endpoint + self.companyid_param, 'GET', params=params) if response.status_code == 200: response_dict = xmltodict.parse(response.text, attr_prefix='', cdata_key='', dict_constructor=dict) - response_table = Table(response_dict['response']['broadcasts']['broadcast']) - response_table.unpack_dict('campaign') - broadcast_table.concat(response_table) - page_count = int(response_dict['response']['broadcasts']['page_count']) + response_table = Table(response_dict['response'][endpoint][data_key]) + for col in cols_to_unpack: + response_table.unpack_dict(col) + final_table.concat(response_table) + avail_pages = int(response_dict['response'][endpoint]['page_count']) + req_pages = math.ceil(limit/page_limit) + pages_to_get = avail_pages if avail_pages < req_pages else req_pages i = 2 - while i <= page_count: + while i <= pages_to_get: page_params = params + f'&page={str(i)}' - response = self.client.request('broadcasts' + self.companyid_param, 'GET', + logger.info(f'Fetching rows {str(i*page_limit)} - {str((i+1)*page_limit)} ' + f'of {limit}') + response = self.client.request(endpoint + self.companyid_param, 'GET', params=page_params) response_dict = xmltodict.parse(response.text, attr_prefix='', cdata_key='', dict_constructor=dict) - response_table = Table(response_dict['response']['broadcasts']['broadcast']) - broadcast_table.concat(response_table) + response_table = Table(response_dict['response'][endpoint][data_key]) + final_table.concat(response_table) i += 1 else: error = f'Response Code {str(response.status_code)}' @@ -65,4 +69,10 @@ def get_broadcasts(self, start_time=None, end_time=None, status=None, campaign_i error += '\n' + error_html.p.next raise HTTPError(error) - return broadcast_table + return final_table + + def get_broadcasts(self, start_time=None, end_time=None, status=None, campaign_id=None, + limit=None): + params = '' + return self.mc_get_request(endpoint='broadcasts', data_key='broadcast', + params=params, cols_to_unpack=['campaign'], limit=limit) From 6574c13306f6c319eeb37101664d47011457a7b2 Mon Sep 17 00:00:00 2001 From: Corma Martinez del Rio <66973815+cmdelrio@users.noreply.github.com> Date: Wed, 17 Aug 2022 16:56:06 -0600 Subject: [PATCH 6/6] Add annotation --- parsons/mobilecommons/mobilecommons.py | 54 +++++++++++++++++++++++--- 1 file changed, 49 insertions(+), 5 deletions(-) diff --git a/parsons/mobilecommons/mobilecommons.py b/parsons/mobilecommons/mobilecommons.py index 9d294cfb8d..dc510ae7ad 100644 --- a/parsons/mobilecommons/mobilecommons.py +++ b/parsons/mobilecommons/mobilecommons.py @@ -33,23 +33,54 @@ def __init__(self, username=None, password=None, companyid=None): self.companyid_param = f'?company={companyid}' if companyid else '' self.client = APIConnector(uri=MC_URI, auth=(self.username,self.password)) - def mc_get_request(self, endpoint, data_key, params, cols_to_unpack, limit): + def mc_get_request(self, endpoint, data_key, params, elements_to_unpack, limit): + """ + A function for GET requests that handles MobileCommons xml responses and pagination + + `Args:` + endpoint: str + The endpoint, which will be appended to the base URL for each request + data_key: str + The key used to extract the desired data from the response dictionary derived from + the xml response + params: str + Parameters to be passed into GET request + elements_to_unpack: list + A list of elements that contain dictionaries to be unpacked into new columns in the + final table + limit: int + The maximum number of rows to return + `Returns:` + Parsons table with requested data + """ + + # Create a table to compile results from different pages in final_table = Table() + # Max page_limit is 1000 for MC page_limit = 1000 if limit > 1000 or limit is None else limit - logger.info(f'Working on fetching first {page_limit} rows. ' - f'Each 1000 rows will take aproximately 40 seconds to fetch.') params += f'limit={str(page_limit)}' + + logger.info(f'Working on fetching first {page_limit} rows. This can take a long time.' + f'Each 1000 rows can take between 30-60 seconds to fetch.') + response = self.client.request(endpoint + self.companyid_param, 'GET', params=params) + + # If good response, compile data into final_table if response.status_code == 200: + # Parse xml to nested dictionary and load to parsons table response_dict = xmltodict.parse(response.text, attr_prefix='', cdata_key='', dict_constructor=dict) response_table = Table(response_dict['response'][endpoint][data_key]) - for col in cols_to_unpack: + # Unpack any specified elements + for col in elements_to_unpack: response_table.unpack_dict(col) + # Append to final table final_table.concat(response_table) + # Check to see if there are more pages and determine how many to retrieve avail_pages = int(response_dict['response'][endpoint]['page_count']) req_pages = math.ceil(limit/page_limit) pages_to_get = avail_pages if avail_pages < req_pages else req_pages + # Go fetch other pages of data i = 2 while i <= pages_to_get: page_params = params + f'&page={str(i)}' @@ -62,6 +93,7 @@ def mc_get_request(self, endpoint, data_key, params, cols_to_unpack, limit): response_table = Table(response_dict['response'][endpoint][data_key]) final_table.concat(response_table) i += 1 + # If initial response is bad, raise error else: error = f'Response Code {str(response.status_code)}' error_html = BeautifulSoup(response.text, features='html.parser') @@ -73,6 +105,18 @@ def mc_get_request(self, endpoint, data_key, params, cols_to_unpack, limit): def get_broadcasts(self, start_time=None, end_time=None, status=None, campaign_id=None, limit=None): + """ + Retrieve broadcast data + + `Args:` + :param start_time: + :param end_time: + :param status: + :param campaign_id: + :param limit: + :return: + """ + # Still working on compiling the params, but this function does work atm with no params specified params = '' return self.mc_get_request(endpoint='broadcasts', data_key='broadcast', - params=params, cols_to_unpack=['campaign'], limit=limit) + params=params, elements_to_unpack=['campaign'], limit=limit)