diff --git a/.github/workflows/test-linux-windows.yml b/.github/workflows/test-linux-windows.yml index 00e13bb748..c68df03375 100644 --- a/.github/workflows/test-linux-windows.yml +++ b/.github/workflows/test-linux-windows.yml @@ -44,4 +44,4 @@ jobs: run: | # E203 and W503 don't work well with black flake8 parsons/ test/ useful_resources/ - black --check parsons/ test/ useful_resources/ + black --diff parsons/ test/ useful_resources/ diff --git a/.github/workflows/tests-mac.yml b/.github/workflows/tests-mac.yml index 578765c8fe..7d5595eda1 100644 --- a/.github/workflows/tests-mac.yml +++ b/.github/workflows/tests-mac.yml @@ -44,4 +44,4 @@ jobs: run: | # E203 and W503 don't work well with black flake8 parsons/ test/ useful_resources/ - black --check parsons/ test/ useful_resources/ + black --diff parsons/ test/ useful_resources/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0c4ead752e..55abef4544 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,6 +8,6 @@ repos: '--max-line-length=100' ] - repo: https://github.com/psf/black - rev: 24.2.0 + rev: 24.3.0 hooks: - id: black diff --git a/docs/bluelink.rst b/docs/bluelink.rst deleted file mode 100644 index 985540a62a..0000000000 --- a/docs/bluelink.rst +++ /dev/null @@ -1,69 +0,0 @@ -Bluelink -============= - -******** -Overview -******** - -`Bluelink `_ is an online tool for connecting various `digital software tools `_ used by campaigns and movement groups in the political and non-profit space so you can sync data between them. This integration currently supports sending your structured person data and related tags to Bluelink via the `Bluelink Webhook API `_, after which you can use Bluelink's UI to send to any of their `supported tools `_. If you don't see a tool you would like to connect to, please reach out at hello@bluelink.org to ask them to add it. - -.. note:: - Authentication - If you don't have a Bluelink account please complete the `form `_ on the Bluelink website or email them at hello@bluelink.org. To get connection credentials select `Bluelink Webhook `_ from the apps menu. If you don't see this option, you may need to ask an account administrator to do this step for you. - - The credentials are automatically embedded into a one time secret link in case they need to be sent to you. Open the link to access the user and password. - -========== -Quickstart -========== - -To instantiate a class, you can either pass in the user and password token as arguments or set them in the -BLUELINK_WEBHOOK_USER and BLUELINK_WEBHOOK_PASSWORD environment variables. - -.. code-block:: python - - from parsons.bluelink import Bluelink - - # First approach: Use API credentials via environmental variables - bluelink = Bluelink() - - # Second approach: Pass API credentials as arguments - bluelink = Bluelink('username', 'password') - -You can upsert person data by directly using a BluelinkPerson object: - -.. code-block:: python - - from parsons.bluelink import Bluelink, BluelinkPerson, BluelinkIdentifier - - # create the person object - person = BluelinkPerson(identifiers=[BluelinkIdentifier(source="SOURCE_VENDOR", identifier="ID")], given_name="Jane", family_name="Doe") - - # use the bluelink connector to upsert - source = "MY_ORG_NAME" - bluelink.upsert_person(source, person) - -You can bulk upsert person data via a Parsons Table by providing a function that takes a row and outputs a BluelinkPerson: - -.. code-block:: python - - from parsons.bluelink import Bluelink, BluelinkPerson, BluelinkIdentifier - - # a function that takes a row and returns a BluelinkPerson - def row_to_person(row): - return BluelinkPerson(identifiers=[BluelinkIdentifier(source="SOURCE_VENDOR", identifier=row["id"])], - given_name=row["firstName"], family_name=row["lastName"]) - - # a parsons table filled with person data - parsons_tbl = get_data() - - # call bulk_upsert_person - source = "MY_ORG_NAME" - bluelink.bulk_upsert_person(source, parsons_tbl, row_to_person) - -*** -API -*** - -.. autoclass :: parsons.bluelink.Bluelink - :inherited-members: diff --git a/docs/index.rst b/docs/index.rst index b3cba40fcf..bb057fbca3 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -189,7 +189,6 @@ Indices and tables azure bill_com bloomerang - bluelink box braintree capitolcanary diff --git a/parsons/__init__.py b/parsons/__init__.py index 6762e54e2d..ebc743e37f 100644 --- a/parsons/__init__.py +++ b/parsons/__init__.py @@ -39,7 +39,6 @@ ("parsons.azure.azure_blob_storage", "AzureBlobStorage"), ("parsons.bill_com.bill_com", "BillCom"), ("parsons.bloomerang.bloomerang", "Bloomerang"), - ("parsons.bluelink", "Bluelink"), ("parsons.box.box", "Box"), ("parsons.braintree.braintree", "Braintree"), ("parsons.capitol_canary.capitol_canary", "CapitolCanary"), diff --git a/parsons/action_kit/action_kit.py b/parsons/action_kit/action_kit.py index 17bdc4f1db..35b5db3383 100644 --- a/parsons/action_kit/action_kit.py +++ b/parsons/action_kit/action_kit.py @@ -175,6 +175,8 @@ def update_user(self, user_id, **kwargs): resp = self.conn.patch(self._base_endpoint("user", user_id), data=json.dumps(kwargs)) logger.info(f"{resp.status_code}: {user_id}") + return resp + def get_event(self, event_id): """Get an event. diff --git a/parsons/bluelink/__init__.py b/parsons/bluelink/__init__.py deleted file mode 100644 index d3c77bf6f1..0000000000 --- a/parsons/bluelink/__init__.py +++ /dev/null @@ -1,21 +0,0 @@ -from parsons.bluelink.bluelink import Bluelink -from parsons.bluelink.person import ( - BluelinkPerson, - BluelinkEmail, - BluelinkAddress, - BluelinkPhone, - BluelinkIdentifier, - BluelinkTag, - BluelinkScore, -) - -__all__ = [ - "Bluelink", - "BluelinkPerson", - "BluelinkEmail", - "BluelinkAddress", - "BluelinkPhone", - "BluelinkIdentifier", - "BluelinkTag", - "BluelinkScore", -] diff --git a/parsons/bluelink/bluelink.py b/parsons/bluelink/bluelink.py deleted file mode 100644 index 93e2f933d0..0000000000 --- a/parsons/bluelink/bluelink.py +++ /dev/null @@ -1,84 +0,0 @@ -from parsons.utilities.api_connector import APIConnector -from parsons.utilities import check_env -from parsons.bluelink.person import BluelinkPerson -import logging -import json - -logger = logging.getLogger(__name__) - -API_URL = "https://api.bluelink.org/webhooks/" - - -class Bluelink: - """ - Instantiate a Bluelink connector. - Allows for a simple method of inserting person data to Bluelink via a webhook. - # see: https://bluelinkdata.github.io/docs/BluelinkApiGuide#webhook - - `Args:`: - user: str - Bluelink webhook user name. - password: str - Bluelink webhook password. - """ - - def __init__(self, user=None, password=None): - self.user = check_env.check("BLUELINK_WEBHOOK_USER", user) - self.password = check_env.check("BLUELINK_WEBHOOK_PASSWORD", password) - self.headers = { - "Content-Type": "application/json", - } - self.api_url = API_URL - self.api = APIConnector(self.api_url, auth=(self.user, self.password), headers=self.headers) - - def upsert_person(self, source, person=None): - """ - Upsert a BluelinkPerson object into Bluelink. - Rows will update, as opposed to being inserted, if an existing person record in - Bluelink has a matching BluelinkIdentifier (same source and id) as the BluelinkPerson object - passed into this function. - - `Args:` - source: str - String to identify that the data came from your system. For example, - your company name. - person: BluelinkPerson - A BluelinkPerson object. - Will be inserted to Bluelink, or updated if a matching record is found. - `Returns:` - int - An http status code from the http post request to the Bluelink webhook. - """ - data = {"source": source, "person": person} - jdata = json.dumps( - data, - default=lambda o: {k: v for k, v in o.__dict__.items() if v is not None}, - ) - resp = self.api.post_request(url=self.api_url, data=jdata) - return resp - - def bulk_upsert_person(self, source, tbl, row_to_person): - """ - Upsert all rows into Bluelink, using the row_to_person function to - transform rows to BluelinkPerson objects. - - `Args:` - source: str - String to identify that the data came from your system. - For example, your company name. - tbl: Table - A parsons Table that represents people data. - row_to_person: Callable[[dict],BluelinkPerson] - A function that takes a dict representation of a row from the passed in tbl - and returns a BluelinkPerson object. - - `Returns:` - list[int] - A list of https response status codes, one response for each row in the table. - """ - people = BluelinkPerson.from_table(tbl, row_to_person) - responses = [] - for person in people: - response = self.upsert_person(source, person) - responses.append(response) - return responses diff --git a/parsons/bluelink/person.py b/parsons/bluelink/person.py deleted file mode 100644 index f808c090f1..0000000000 --- a/parsons/bluelink/person.py +++ /dev/null @@ -1,280 +0,0 @@ -import logging -import json - -logger = logging.getLogger(__name__) - - -class BluelinkPerson(object): - """ - Instantiate BluelinkPerson Class. - Used for to upserting via Bluelink connector. - See: https://bluelinkdata.github.io/docs/BluelinkApiGuide#person-object - - `Args:` - identifiers: list[BluelinkIdentifier] - A list of BluelinkIdentifier objects. - A BluelinkPerson must have at least 1 identifier. - given_name: str - First name / given name. - family_name: str - Last name / family name. - phones: list[BluelinkPhone] - A list of BluelinkPhone objects representing phone numbers. - emails: list[BluelinkEmail] - A list of BluelinkEmail objects representing email addresses. - addresses: list[BluelinkAddress] - A list of BluelinkAddress objects representing postal addresses. - tags: list[BluelinkTag] - Simple tags that apply to the person, eg DONOR. - employer: str - Name of the persons employer. - employer_address: BluelinkAddress - BluelinkAddress of the persons employer. - occupation: str - Occupation. - scores: list[BluelinkScore] - List of BluelinkScore objects. Scores are numeric scores, ie partisanship model. - birthdate: str - ISO 8601 formatted birth date: 'YYYY-MM-DD' - details: dict - additional custom data. must be json serializable. - """ - - def __init__( - self, - identifiers, - given_name=None, - family_name=None, - phones=None, - emails=None, - addresses=None, - tags=None, - employer=None, - employer_address=None, - occupation=None, - scores=None, - birthdate=None, - details=None, - ): - - if not identifiers: - raise Exception( - "BluelinkPerson requires list of BluelinkIdentifiers with " - "at least 1 BluelinkIdentifier" - ) - - self.identifiers = identifiers - self.addresses = addresses - self.emails = emails - self.phones = phones - self.tags = tags - self.scores = scores - - self.given_name = given_name - self.family_name = family_name - - self.employer = employer - self.employer_address = employer_address - self.occupation = occupation - self.birthdate = birthdate - self.details = details - - def __json__(self): - """The json str representation of this BluelinkPerson object""" - return json.dumps(self, default=lambda obj: obj.__dict__) - - def __eq__(self, other): - """A quick and dirty equality check""" - dself = json.loads(self.__json__()) - dother = json.loads(other.__json__()) - return dself == dother - - def __repr__(self): - return self.__json__() - - @staticmethod - def from_table(tbl, dict_to_person): - """ - Return a list of BluelinkPerson objects from a Parsons Table. - - `Args:` - tbl: Table - A parsons Table. - dict_to_person: Callable[[dict],BluelinkPerson] - A function that takes a dictionary representation of a table row, - and returns a BluelinkPerson. - `Returns:` - list[BluelinkPerson] - A list of BluelinkPerson objects. - """ - return [dict_to_person(row) for row in tbl] - - -class BluelinkIdentifier(object): - """ - Instantiate an BluelinkIdentifier object. - BluelinkIdentifier is necessary for updating BluelinkPerson records. - - `Args:` - source: str - External system to which this ID belongs, e.g., “VAN:myCampaign”. - Bluelink has standardized strings for source. Using these will - allow Bluelink to correctly understand the external IDs you add. - source (unlike identifier) is case insensitive. - examples: BLUELINK, PDI, SALESFORCE, VAN:myCampaign, VAN:myVoters - identifier: str - Case-sensitive ID in the external system. - details: dict - dictionary of custom fields. must be serializable to json. - """ - - def __init__(self, source, identifier, details=None): - self.source = source - self.identifier = identifier - self.details = details - - -class BluelinkEmail(object): - """ - Instantiate an BluelinkEmail object. - - `Args:` - address: str - An email address. ie "user@example.com" - primary: bool - True if this is known to be the primary email. - type: str - Type, eg: "personal", "work" - status: str - One of "Potential", "Subscribed", "Unsubscribed", "Bouncing", or "Spam Complaints" - """ - - def __init__(self, address, primary=None, type=None, status=None): - self.address = address - self.primary = primary - self.type = type - self.status = status - - -class BluelinkAddress(object): - """ - Instantiate an BluelinkAddress object. - - `Args`: - address_lines: list[str] - A list of street address lines. - city: str - City or other locality. - state: str - State in ISO 3166-2. - postal_code: str - Zip or other postal code. - country: str - ISO 3166-1 Alpha-2 country code. - type: str - The type. ie: "home", "mailing". - venue: str - The venue name, if relevant. - status: str - A value representing the status of the address. "Potential", "Verified" or "Bad" - """ - - def __init__( - self, - address_lines=None, - city=None, - state=None, - postal_code=None, - country=None, - type=None, - venue=None, - status=None, - ): - - self.address_lines = address_lines or [] - self.city = city - self.state = state - self.postal_code = postal_code - self.country = country - - self.type = type - self.venue = venue - self.status = status - - -class BluelinkPhone(object): - """ - Instantiate a BluelinkPhone object. - - `Args:` - number: str - A phone number. May or may not include country code. - primary: bool - True if this is known to be the primary phone. - description: str - Free for description. - type: str - Type, eg: "Home", "Work", "Mobile" - country: str - ISO 3166-1 Alpha-2 country code. - sms_capable: bool - True if this number can accept SMS. - do_not_call: bool - True if this number is on the US FCC Do Not Call Registry. - details: dict - Additional data dictionary. Must be json serializable. - """ - - def __init__( - self, - number, - primary=None, - description=None, - type=None, - country=None, - sms_capable=None, - do_not_call=None, - details=None, - ): - self.number = number - self.primary = primary - self.description = description - self.type = type - self.country = country - self.sms_capable = sms_capable - self.do_not_call = do_not_call - self.details = details - - -class BluelinkTag(object): - """ - Instantiate a BluelinkTag object. - - `Args:` - tag: str - A tag string; convention is either a simple string - or a string with a prefix separated by a colon, e.g., “DONOR:GRASSROOTS” - """ - - def __init__(self, tag): - self.tag = tag - - -class BluelinkScore(object): - """ - Instantiate a score object. - Represents some kind of numeric score. - - `Args`: - score: float - Numeric score. - score_type: str - Type, eg: "Partisanship model". - source: str - Original source of this score. - """ - - def __init__(self, score, score_type, source): - self.score = score - self.score_type = score_type - self.source = source diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py index fecb15ed2f..9b2f6330c5 100644 --- a/parsons/google/google_bigquery.py +++ b/parsons/google/google_bigquery.py @@ -11,6 +11,7 @@ from google.cloud import bigquery, exceptions from google.cloud.bigquery import dbapi from google.cloud.bigquery.job import LoadJobConfig +from google.oauth2.credentials import Credentials from parsons.databases.database_connector import DatabaseConnector from parsons.databases.table import BaseTable @@ -143,7 +144,7 @@ class GoogleBigQuery(DatabaseConnector): def __init__( self, - app_creds=None, + app_creds: Optional[Union[str, dict, Credentials]] = None, project=None, location=None, client_options: dict = { @@ -156,7 +157,11 @@ def __init__( ): self.app_creds = app_creds - setup_google_application_credentials(app_creds) + if isinstance(app_creds, Credentials): + self.credentials = app_creds + else: + self.credentials = None + setup_google_application_credentials(app_creds) self.project = project self.location = location @@ -185,6 +190,7 @@ def client(self): project=self.project, location=self.location, client_options=self.client_options, + credentials=self.credentials, ) return self._client diff --git a/parsons/google/google_sheets.py b/parsons/google/google_sheets.py index 67deaefc1c..1ab1b2dd5b 100644 --- a/parsons/google/google_sheets.py +++ b/parsons/google/google_sheets.py @@ -3,7 +3,7 @@ import logging from parsons.etl.table import Table -from parsons.google.utitities import setup_google_application_credentials +from parsons.google.utitities import setup_google_application_credentials, hexavigesimal import gspread from google.oauth2.service_account import Credentials @@ -293,6 +293,63 @@ def append_to_sheet( sheet.update_cells(cells, value_input_option=value_input_option) logger.info(f"Appended {table.num_rows} rows to worksheet.") + def paste_data_in_sheet( + self, spreadsheet_id, table, worksheet=0, header=True, startrow=0, startcol=0 + ): + """ + Pastes data from a Parsons table to a Google sheet. Note that this may overwrite + presently existing data. This function is useful for adding data to a subsection + if an existint sheet that will have other existing data - constrast to + `overwrite_sheet` (which will fully replace any existing data) and `append_to_sheet` + (whuch sticks the data only after all other existing data). + + `Args:` + spreadsheet_id: str + The ID of the spreadsheet (Tip: Get this from the spreadsheet URL). + table: obj + Parsons table + worksheet: str or int + The index or the title of the worksheet. The index begins with 0. + header: bool + Whether or not the header row gets pasted with the data. + startrow: int + Starting row position of pasted data. Counts from 0. + startcol: int + Starting column position of pasted data. Counts from 0. + """ + sheet = self._get_worksheet(spreadsheet_id, worksheet) + + number_of_columns = len(table.columns) + number_of_rows = table.num_rows + 1 if header else table.num_rows + + if not number_of_rows or not number_of_columns: # No data to paste + logger.warning( + f"No data available to paste, table size " + f"({number_of_rows}, {number_of_columns}). Skipping." + ) + return + + # gspread uses ranges like "C3:J7", so we need to convert to this format + data_range = ( + hexavigesimal(startcol + 1) + + str(startrow + 1) + + ":" + + hexavigesimal(startcol + number_of_columns) + + str(startrow + number_of_rows) + ) + + # Unpack data. Hopefully this is small enough for memory + data = [[]] * table.num_rows + for row_num, row in enumerate(table.data): + data[row_num] = list(row) + + if header: + sheet.update(data_range, [table.columns] + data) + else: + sheet.update(data_range, data) + + logger.info(f"Pasted data to {data_range} in worksheet.") + def overwrite_sheet( self, spreadsheet_id, table, worksheet=0, user_entered_value=False, **kwargs ): diff --git a/parsons/google/utitities.py b/parsons/google/utitities.py index abda0bf212..75cdf01de0 100644 --- a/parsons/google/utitities.py +++ b/parsons/google/utitities.py @@ -22,3 +22,29 @@ def setup_google_application_credentials( creds_path = credentials os.environ[env_var_name] = creds_path + + +def hexavigesimal(n: int) -> str: + """ + Converts an integer value to the type of strings you see on spreadsheets + (A, B,...,Z, AA, AB, ...). + + Code based on + https://stackoverflow.com/questions/16190452/converting-from-number-to-hexavigesimal-letters + + `Args:` + n: int + A positive valued integer. + + `Returns:` + str + The hexavigeseimal representation of n + """ + if n < 1: + raise ValueError(f"This function only works for positive integers. Provided value {n}.") + + chars = "" + while n != 0: + chars = chr((n - 1) % 26 + 65) + chars # 65 makes us start at A + n = (n - 1) // 26 + return chars diff --git a/parsons/ngpvan/email.py b/parsons/ngpvan/email.py new file mode 100644 index 0000000000..1e5c55c079 --- /dev/null +++ b/parsons/ngpvan/email.py @@ -0,0 +1,150 @@ +from parsons.etl.table import Table +import logging + +logger = logging.getLogger(__name__) + + +class Email(object): + """ + Instantiate the Email class. + + You can find the docs for the NGP VAN Email API here: + https://docs.ngpvan.com/reference/email-overview + """ + + def __init__(self, van_connection): + + self.connection = van_connection + + def get_emails(self, ascending: bool = True) -> Table: + """ + Get emails. + + `Args:` + ascending : Bool + sorts results in ascending or descending order + for the dateModified field. Defaults to True (ascending). + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + if ascending: + params = { + "$orderby": "dateModified asc", + } + if not ascending: + params = { + "$orderby": "dateModified desc", + } + + tbl = Table(self.connection.get_request("email/messages", params=params)) + logger.debug(f"Found {tbl.num_rows} emails.") + return tbl + + def get_email(self, email_id: int, expand: bool = True) -> Table: + """ + Get an email. + + Note that it takes some time for the system to aggregate opens and click-throughs, + so data can be delayed up to 15 minutes. + + `Args:` + email_id : int + The email id. + expand : bool + Optional; expands the email message to include the email content and + statistics. Defaults to True. + + `Returns:` + dict + """ + + params = { + "$expand": ( + "emailMessageContent, EmailMessageContentDistributions" + if expand + else None + ), + } + + r = self.connection.get_request(f"email/message/{email_id}", params=params) + logger.debug(f"Found email {email_id}.") + return r + + def get_email_stats(self) -> Table: + """ + Get stats for all emails, aggregating any A/B tests. + + `Args:` + emails : list + A list of email message details. + + `Returns:` + Parsons Table + See :ref:`parsons-table` for output options. + """ + + email_list = [] + + final_email_list = [] + + emails = self.get_emails() + + foreign_message_ids = [email["foreignMessageId"] for email in emails] + + for fmid in foreign_message_ids: + email = self.get_email(fmid) + email_list.append(email) + + for email in email_list: + d = {} + d["name"] = email["name"] + d["createdBy"] = email["createdBy"] + d["dateCreated"] = email["dateCreated"] + d["dateModified"] = email["dateModified"] + d["dateScheduled"] = email["dateScheduled"] + d["foreignMessageId"] = email["foreignMessageId"] + d["recipientCount"] = 0 + d["bounceCount"] = 0 + d["contributionCount"] = 0 + d["contributionTotal"] = 0 + d["formSubmissionCount"] = 0 + d["linksClickedCount"] = 0 + d["machineOpenCount"] = 0 + d["openCount"] = 0 + d["unsubscribeCount"] = 0 + try: + for i in email["emailMessageContent"]: + d["recipientCount"] += i["emailMessageContentDistributions"][ + "recipientCount" + ] + d["bounceCount"] += i["emailMessageContentDistributions"][ + "bounceCount" + ] + d["contributionCount"] += i["emailMessageContentDistributions"][ + "contributionCount" + ] + d["contributionTotal"] += i["emailMessageContentDistributions"][ + "contributionTotal" + ] + d["formSubmissionCount"] += i["emailMessageContentDistributions"][ + "formSubmissionCount" + ] + d["linksClickedCount"] += i["emailMessageContentDistributions"][ + "linksClickedCount" + ] + d["machineOpenCount"] += i["emailMessageContentDistributions"][ + "machineOpenCount" + ] + d["openCount"] += i["emailMessageContentDistributions"]["openCount"] + d["unsubscribeCount"] += i["emailMessageContentDistributions"][ + "unsubscribeCount" + ] + except TypeError as e: + logger.info(str(e)) + pass + + final_email_list.append(d) + + return Table(final_email_list) diff --git a/parsons/ngpvan/van.py b/parsons/ngpvan/van.py index c57bf835cb..e5f6897ce5 100644 --- a/parsons/ngpvan/van.py +++ b/parsons/ngpvan/van.py @@ -1,5 +1,6 @@ import logging from parsons.ngpvan.events import Events +from parsons.ngpvan.email import Email from parsons.ngpvan.van_connector import VANConnector from parsons.ngpvan.people import People from parsons.ngpvan.saved_lists import SavedLists, Folders, ExportJobs @@ -24,6 +25,7 @@ class VAN( People, Events, + Email, SavedLists, PrintedLists, Folders, diff --git a/requirements-dev.txt b/requirements-dev.txt index 0bee9cdfa2..4ef91afbcc 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ # Testing Requirements -black==24.2.0 +black==24.3.0 Flake8-pyproject==1.2.3 flake8==7.0.0 pytest-datadir==1.5.0 diff --git a/requirements.txt b/requirements.txt index 43aa054ce4..d911a8f734 100644 --- a/requirements.txt +++ b/requirements.txt @@ -24,7 +24,7 @@ newmode==0.1.6 oauth2client==4.1.3 paramiko==3.4.0 petl==1.7.15 -psycopg2-binary==2.9.3 +psycopg2-binary==2.9.9 PyGitHub==1.51 PyJWT==2.4.0 # Otherwise `import jwt` would refer to python-jwt package python-dateutil==2.8.2 @@ -33,7 +33,7 @@ requests_oauthlib==1.3.0 simple-salesforce==1.11.6 simplejson==3.16.0 slackclient==1.3.0 -SQLAlchemy==1.3.23 +sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0 # Prefect does not work with 1.4.33 and >3.0.0 has breaking changes suds-py3==1.3.4.0 surveygizmo==1.2.3 twilio==8.2.1 @@ -45,4 +45,4 @@ xmltodict==0.11.0 # TODO Remove when we have a TMC-specific Docker image jinja2>=3.0.2 selenium==3.141.0 -us==3.1.1 \ No newline at end of file +us==3.1.1 diff --git a/setup.py b/setup.py index 04655af48e..18451928eb 100644 --- a/setup.py +++ b/setup.py @@ -40,12 +40,15 @@ def main(): "oauth2client", "validate-email", ], - "mysql": ["mysql-connector-python", "SQLAlchemy"], + "mysql": [ + "mysql-connector-python", + "sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0", + ], "newmode": ["newmode"], "ngpvan": ["suds-py3"], "mobilecommons": ["bs4"], - "postgres": ["psycopg2-binary", "SQLAlchemy"], - "redshift": ["boto3", "psycopg2-binary", "SQLAlchemy"], + "postgres": ["psycopg2-binary>=2.9.9", "sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0",], + "redshift": ["boto3", "psycopg2-binary>=2.9.9", "sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0",], "s3": ["boto3"], "salesforce": ["simple-salesforce"], "sftp": ["paramiko"], @@ -89,7 +92,7 @@ def main(): ], python_requires=">=3.7.0,<3.12.0", long_description=long_description, - long_description_content_type='text/markdown' + long_description_content_type="text/markdown", ) diff --git a/test/test_action_kit.py b/test/test_action_kit.py index 378d6c55f3..6a0b3fb454 100644 --- a/test/test_action_kit.py +++ b/test/test_action_kit.py @@ -89,12 +89,14 @@ def test_update_user(self): type(resp_mock.patch()).status_code = mock.PropertyMock(return_value=202) self.actionkit.conn = resp_mock - self.actionkit.update_user(123, last_name="new name") + res = self.actionkit.update_user(123, last_name="new name") self.actionkit.conn.patch.assert_called_with( "https://domain.actionkit.com/rest/v1/user/123/", data=json.dumps({"last_name": "new name"}), ) + assert res.status_code == 202 + def test_update_event(self): # Test update event diff --git a/test/test_bluelink/test_bluelink.py b/test/test_bluelink/test_bluelink.py deleted file mode 100644 index 5c15ea5f70..0000000000 --- a/test/test_bluelink/test_bluelink.py +++ /dev/null @@ -1,117 +0,0 @@ -import unittest -import requests_mock -from parsons import Table, Bluelink -from parsons.bluelink import BluelinkPerson, BluelinkIdentifier, BluelinkEmail - - -class TestBluelink(unittest.TestCase): - @requests_mock.Mocker() - def setUp(self, m): - self.bluelink = Bluelink("fake_user", "fake_password") - - @staticmethod - def row_to_person(row): - """ - dict -> BluelinkPerson - Transforms a parsons Table row to a BluelinkPerson. - This function is passed into bulk_upsert_person along with a Table - """ - email = row["email"] - return BluelinkPerson( - identifiers=[ - BluelinkIdentifier(source="FAKESOURCE", identifier=email), - ], - emails=[BluelinkEmail(address=email, primary=True)], - family_name=row["family_name"], - given_name=row["given_name"], - ) - - @staticmethod - def get_table(): - return Table( - [ - { - "given_name": "Bart", - "family_name": "Simpson", - "email": "bart@springfield.net", - }, - { - "given_name": "Homer", - "family_name": "Simpson", - "email": "homer@springfield.net", - }, - ] - ) - - @requests_mock.Mocker() - def test_bulk_upsert_person(self, m): - """ - This function demonstrates how to use a "row_to_person" function to bulk - insert people using a Table as the data source - """ - # Mock POST requests to api - m.post(self.bluelink.api_url) - - # get data as a parsons Table - tbl = self.get_table() - - # String to identify that the data came from your system. For example, your company name. - source = "BLUELINK-PARSONS-TEST" - - # call bulk_upsert_person - # passing in the source, the Table, and the function that maps a Table row -> BluelinkPerson - self.bluelink.bulk_upsert_person(source, tbl, self.row_to_person) - - @requests_mock.Mocker() - def test_upsert_person(self, m): - """ - This function demonstrates how to insert a single BluelinkPerson record - """ - # Mock POST requests to api - m.post(self.bluelink.api_url) - - # create a BluelinkPerson object - # The BluelinkIdentifier is pretending that the user can be - # identified in SALESFORCE with FAKE_ID as her id - person = BluelinkPerson( - identifiers=[BluelinkIdentifier(source="SALESFORCE", identifier="FAKE_ID")], - given_name="Jane", - family_name="Doe", - emails=[BluelinkEmail(address="jdoe@example.com", primary=True)], - ) - - # String to identify that the data came from your system. For example, your company name. - source = "BLUELINK-PARSONS-TEST" - - # call upsert_person - self.bluelink.upsert_person(source, person) - - def test_table_to_people(self): - """ - Test transforming a parsons Table -> list[BluelinkPerson] - """ - # setup - tbl = self.get_table() - - # function under test - actual_people = BluelinkPerson.from_table(tbl, self.row_to_person) - - # expected: - person1 = BluelinkPerson( - identifiers=[ - BluelinkIdentifier(source="FAKESOURCE", identifier="bart@springfield.net") - ], - emails=[BluelinkEmail(address="bart@springfield.net", primary=True)], - family_name="Simpson", - given_name="Bart", - ) - person2 = BluelinkPerson( - identifiers=[ - BluelinkIdentifier(source="FAKESOURCE", identifier="homer@springfield.net") - ], - emails=[BluelinkEmail(address="homer@springfield.net", primary=True)], - family_name="Simpson", - given_name="Homer", - ) - expected_people = [person1, person2] - self.assertEqual(actual_people, expected_people) diff --git a/test/test_google/test_google_sheets.py b/test/test_google/test_google_sheets.py index d7672b76a7..ed6811bb6a 100644 --- a/test/test_google/test_google_sheets.py +++ b/test/test_google/test_google_sheets.py @@ -123,6 +123,82 @@ def test_append_user_entered_to_spreadsheet(self): self.assertEqual(formula_vals[0], "27") self.assertEqual(formula_vals[1], "Budapest") + def test_paste_data_in_sheet(self): + # Testing if we can paste data to a spreadsheet + # TODO: there's probably a smarter way to test this code + self.google_sheets.add_sheet(self.spreadsheet_id, "PasteDataSheet") + + paste_table1 = Table( + [ + {"col1": 1, "col2": 2}, + {"col1": 5, "col2": 6}, + ] + ) + paste_table2 = Table( + [ + {"col3": 3, "col4": 4}, + {"col3": 7, "col4": 8}, + ] + ) + paste_table3 = Table( + [ + {"col1": 9, "col2": 10}, + {"col1": 13, "col2": 14}, + ] + ) + paste_table4 = Table( + [ + {"col3": 11, "col4": 12}, + {"col3": 15, "col4": 16}, + ] + ) + + # When we read the spreadsheet, it assumes data is all strings + expected_table = Table( + [ + {"col1": "1", "col2": "2", "col3": "3", "col4": "4"}, + {"col1": "5", "col2": "6", "col3": "7", "col4": "8"}, + {"col1": "9", "col2": "10", "col3": "11", "col4": "12"}, + {"col1": "13", "col2": "14", "col3": "15", "col4": "16"}, + ] + ) + + self.google_sheets.paste_data_in_sheet( + self.spreadsheet_id, + paste_table1, + worksheet="PasteDataSheet", + header=True, + startrow=0, + startcol=0, + ) + self.google_sheets.paste_data_in_sheet( + self.spreadsheet_id, + paste_table2, + worksheet="PasteDataSheet", + header=True, + startrow=0, + startcol=2, + ) + self.google_sheets.paste_data_in_sheet( + self.spreadsheet_id, + paste_table3, + worksheet="PasteDataSheet", + header=False, + startrow=3, + startcol=0, + ) + self.google_sheets.paste_data_in_sheet( + self.spreadsheet_id, + paste_table4, + worksheet="PasteDataSheet", + header=False, + startrow=3, + startcol=2, + ) + + result_table = self.google_sheets.get_worksheet(self.spreadsheet_id, "PasteDataSheet") + self.assertEqual(result_table.to_dicts(), expected_table.to_dicts()) + def test_overwrite_spreadsheet(self): new_table = Table( [ diff --git a/test/test_google/test_utilities.py b/test/test_google/test_utilities.py index 5a6b8c2966..accb506f90 100644 --- a/test/test_google/test_utilities.py +++ b/test/test_google/test_utilities.py @@ -72,3 +72,16 @@ def test_credentials_are_valid_after_double_call(self): actual = fsnd.read() self.assertEqual(self.cred_contents, json.loads(actual)) self.assertEqual(ffst.read(), actual) + + +class TestHexavigesimal(unittest.TestCase): + + def test_returns_A_on_1(self): + self.assertEqual(util.hexavigesimal(1), "A") + + def test_returns_AA_on_27(self): + self.assertEqual(util.hexavigesimal(27), "AA") + + def test_returns_error_on_0(self): + with self.assertRaises(ValueError): + util.hexavigesimal(0) diff --git a/test/test_van/test_email.py b/test/test_van/test_email.py new file mode 100644 index 0000000000..faf9aabcee --- /dev/null +++ b/test/test_van/test_email.py @@ -0,0 +1,111 @@ +import unittest +import os +import requests_mock +from parsons import VAN, Table + + +def assert_matching_tables(table1, table2, ignore_headers=False): + if ignore_headers: + data1 = table1.data + data2 = table2.data + else: + data1 = table1 + data2 = table2 + + if isinstance(data1, Table) and isinstance(data2, Table): + assert data1.num_rows == data2.num_rows + + for r1, r2 in zip(data1, data2): + # Cast both rows to lists, in case they are different types of collections. Must call + # .items() on dicts to compare content of collections + if isinstance(r1, dict): + r1 = r1.items() + if isinstance(r2, dict): + r2 = r2.items() + + assert list(r1) == list(r2) + + +os.environ["VAN_API_KEY"] = "SOME_KEY" + +mock_response = [ + { + "foreignMessageId": "oK2ahdAcEe6F-QAiSCI3lA2", + "name": "Test Email", + "createdBy": "Joe Biden", + "dateCreated": "2024-02-20T13:20:00Z", + "dateScheduled": "2024-02-20T15:55:00Z", + "campaignID": 0, + "dateModified": "2024-02-20T15:54:36.27Z", + "emailMessageContent": None, + }, + { + "foreignMessageId": "rjzc2szzEe6F-QAiSCI3lA2", + "name": "Test Email 2", + "createdBy": "Joe Biden", + "dateCreated": "2024-02-16T12:49:00Z", + "dateScheduled": "2024-02-16T13:29:00Z", + "campaignID": 0, + "dateModified": "2024-02-16T13:29:16.453Z", + "emailMessageContent": None, + }, + { + "foreignMessageId": "_E1AfcnkEe6F-QAiSCI3lA2", + "name": "Test Email 3", + "createdBy": "Joe Biden", + "dateCreated": "2024-02-12T15:26:00Z", + "dateScheduled": "2024-02-13T11:22:00Z", + "campaignID": 0, + "dateModified": "2024-02-13T11:22:28.273Z", + "emailMessageContent": None, + }, + { + "foreignMessageId": "6GTLBsUwEe62YAAiSCIxlw2", + "name": "Test Email 4", + "createdBy": "Joe Biden", + "dateCreated": "2024-02-06T15:47:00Z", + "dateScheduled": "2024-02-07T10:32:00Z", + "campaignID": 0, + "dateModified": "2024-02-07T10:31:55.16Z", + "emailMessageContent": None, + }, + { + "foreignMessageId": "mgTdmcEiEe62YAAiSCIxlw2", + "name": "Test Email 5", + "createdBy": "Joe Biden", + "dateCreated": "2024-02-01T11:55:00Z", + "dateScheduled": "2024-02-01T16:08:00Z", + "campaignID": 0, + "dateModified": "2024-02-01T16:08:10.737Z", + "emailMessageContent": None, + }, +] + + +class TestEmail(unittest.TestCase): + def setUp(self): + self.van = VAN(os.environ["VAN_API_KEY"], db="MyVoters", raise_for_status=False) + + @requests_mock.Mocker() + def test_get_email_messages(self, m): + m.get( + self.van.connection.uri + "email/messages", + json=mock_response, + status_code=200, + ) + + response = self.van.get_emails() + + assert_matching_tables(response, mock_response) + + @requests_mock.Mocker() + def test_get_email_message(self, m): + m.get( + self.van.connection.uri + "email/message/1", + json=mock_response[0], + status_code=200, + ) + + response = self.van.get_email(1) + + assert_matching_tables(response, mock_response[0])