diff --git a/.github/workflows/test-linux-windows.yml b/.github/workflows/test-linux-windows.yml
index 00e13bb748..c68df03375 100644
--- a/.github/workflows/test-linux-windows.yml
+++ b/.github/workflows/test-linux-windows.yml
@@ -44,4 +44,4 @@ jobs:
run: |
# E203 and W503 don't work well with black
flake8 parsons/ test/ useful_resources/
- black --check parsons/ test/ useful_resources/
+ black --diff parsons/ test/ useful_resources/
diff --git a/.github/workflows/tests-mac.yml b/.github/workflows/tests-mac.yml
index 578765c8fe..7d5595eda1 100644
--- a/.github/workflows/tests-mac.yml
+++ b/.github/workflows/tests-mac.yml
@@ -44,4 +44,4 @@ jobs:
run: |
# E203 and W503 don't work well with black
flake8 parsons/ test/ useful_resources/
- black --check parsons/ test/ useful_resources/
+ black --diff parsons/ test/ useful_resources/
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 0c4ead752e..55abef4544 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -8,6 +8,6 @@ repos:
'--max-line-length=100'
]
- repo: https://github.com/psf/black
- rev: 24.2.0
+ rev: 24.3.0
hooks:
- id: black
diff --git a/docs/bluelink.rst b/docs/bluelink.rst
deleted file mode 100644
index 985540a62a..0000000000
--- a/docs/bluelink.rst
+++ /dev/null
@@ -1,69 +0,0 @@
-Bluelink
-=============
-
-********
-Overview
-********
-
-`Bluelink `_ is an online tool for connecting various `digital software tools `_ used by campaigns and movement groups in the political and non-profit space so you can sync data between them. This integration currently supports sending your structured person data and related tags to Bluelink via the `Bluelink Webhook API `_, after which you can use Bluelink's UI to send to any of their `supported tools `_. If you don't see a tool you would like to connect to, please reach out at hello@bluelink.org to ask them to add it.
-
-.. note::
- Authentication
- If you don't have a Bluelink account please complete the `form `_ on the Bluelink website or email them at hello@bluelink.org. To get connection credentials select `Bluelink Webhook `_ from the apps menu. If you don't see this option, you may need to ask an account administrator to do this step for you.
-
- The credentials are automatically embedded into a one time secret link in case they need to be sent to you. Open the link to access the user and password.
-
-==========
-Quickstart
-==========
-
-To instantiate a class, you can either pass in the user and password token as arguments or set them in the
-BLUELINK_WEBHOOK_USER and BLUELINK_WEBHOOK_PASSWORD environment variables.
-
-.. code-block:: python
-
- from parsons.bluelink import Bluelink
-
- # First approach: Use API credentials via environmental variables
- bluelink = Bluelink()
-
- # Second approach: Pass API credentials as arguments
- bluelink = Bluelink('username', 'password')
-
-You can upsert person data by directly using a BluelinkPerson object:
-
-.. code-block:: python
-
- from parsons.bluelink import Bluelink, BluelinkPerson, BluelinkIdentifier
-
- # create the person object
- person = BluelinkPerson(identifiers=[BluelinkIdentifier(source="SOURCE_VENDOR", identifier="ID")], given_name="Jane", family_name="Doe")
-
- # use the bluelink connector to upsert
- source = "MY_ORG_NAME"
- bluelink.upsert_person(source, person)
-
-You can bulk upsert person data via a Parsons Table by providing a function that takes a row and outputs a BluelinkPerson:
-
-.. code-block:: python
-
- from parsons.bluelink import Bluelink, BluelinkPerson, BluelinkIdentifier
-
- # a function that takes a row and returns a BluelinkPerson
- def row_to_person(row):
- return BluelinkPerson(identifiers=[BluelinkIdentifier(source="SOURCE_VENDOR", identifier=row["id"])],
- given_name=row["firstName"], family_name=row["lastName"])
-
- # a parsons table filled with person data
- parsons_tbl = get_data()
-
- # call bulk_upsert_person
- source = "MY_ORG_NAME"
- bluelink.bulk_upsert_person(source, parsons_tbl, row_to_person)
-
-***
-API
-***
-
-.. autoclass :: parsons.bluelink.Bluelink
- :inherited-members:
diff --git a/docs/index.rst b/docs/index.rst
index b3cba40fcf..bb057fbca3 100755
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -189,7 +189,6 @@ Indices and tables
azure
bill_com
bloomerang
- bluelink
box
braintree
capitolcanary
diff --git a/parsons/__init__.py b/parsons/__init__.py
index 6762e54e2d..ebc743e37f 100644
--- a/parsons/__init__.py
+++ b/parsons/__init__.py
@@ -39,7 +39,6 @@
("parsons.azure.azure_blob_storage", "AzureBlobStorage"),
("parsons.bill_com.bill_com", "BillCom"),
("parsons.bloomerang.bloomerang", "Bloomerang"),
- ("parsons.bluelink", "Bluelink"),
("parsons.box.box", "Box"),
("parsons.braintree.braintree", "Braintree"),
("parsons.capitol_canary.capitol_canary", "CapitolCanary"),
diff --git a/parsons/action_kit/action_kit.py b/parsons/action_kit/action_kit.py
index 17bdc4f1db..35b5db3383 100644
--- a/parsons/action_kit/action_kit.py
+++ b/parsons/action_kit/action_kit.py
@@ -175,6 +175,8 @@ def update_user(self, user_id, **kwargs):
resp = self.conn.patch(self._base_endpoint("user", user_id), data=json.dumps(kwargs))
logger.info(f"{resp.status_code}: {user_id}")
+ return resp
+
def get_event(self, event_id):
"""Get an event.
diff --git a/parsons/bluelink/__init__.py b/parsons/bluelink/__init__.py
deleted file mode 100644
index d3c77bf6f1..0000000000
--- a/parsons/bluelink/__init__.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from parsons.bluelink.bluelink import Bluelink
-from parsons.bluelink.person import (
- BluelinkPerson,
- BluelinkEmail,
- BluelinkAddress,
- BluelinkPhone,
- BluelinkIdentifier,
- BluelinkTag,
- BluelinkScore,
-)
-
-__all__ = [
- "Bluelink",
- "BluelinkPerson",
- "BluelinkEmail",
- "BluelinkAddress",
- "BluelinkPhone",
- "BluelinkIdentifier",
- "BluelinkTag",
- "BluelinkScore",
-]
diff --git a/parsons/bluelink/bluelink.py b/parsons/bluelink/bluelink.py
deleted file mode 100644
index 93e2f933d0..0000000000
--- a/parsons/bluelink/bluelink.py
+++ /dev/null
@@ -1,84 +0,0 @@
-from parsons.utilities.api_connector import APIConnector
-from parsons.utilities import check_env
-from parsons.bluelink.person import BluelinkPerson
-import logging
-import json
-
-logger = logging.getLogger(__name__)
-
-API_URL = "https://api.bluelink.org/webhooks/"
-
-
-class Bluelink:
- """
- Instantiate a Bluelink connector.
- Allows for a simple method of inserting person data to Bluelink via a webhook.
- # see: https://bluelinkdata.github.io/docs/BluelinkApiGuide#webhook
-
- `Args:`:
- user: str
- Bluelink webhook user name.
- password: str
- Bluelink webhook password.
- """
-
- def __init__(self, user=None, password=None):
- self.user = check_env.check("BLUELINK_WEBHOOK_USER", user)
- self.password = check_env.check("BLUELINK_WEBHOOK_PASSWORD", password)
- self.headers = {
- "Content-Type": "application/json",
- }
- self.api_url = API_URL
- self.api = APIConnector(self.api_url, auth=(self.user, self.password), headers=self.headers)
-
- def upsert_person(self, source, person=None):
- """
- Upsert a BluelinkPerson object into Bluelink.
- Rows will update, as opposed to being inserted, if an existing person record in
- Bluelink has a matching BluelinkIdentifier (same source and id) as the BluelinkPerson object
- passed into this function.
-
- `Args:`
- source: str
- String to identify that the data came from your system. For example,
- your company name.
- person: BluelinkPerson
- A BluelinkPerson object.
- Will be inserted to Bluelink, or updated if a matching record is found.
- `Returns:`
- int
- An http status code from the http post request to the Bluelink webhook.
- """
- data = {"source": source, "person": person}
- jdata = json.dumps(
- data,
- default=lambda o: {k: v for k, v in o.__dict__.items() if v is not None},
- )
- resp = self.api.post_request(url=self.api_url, data=jdata)
- return resp
-
- def bulk_upsert_person(self, source, tbl, row_to_person):
- """
- Upsert all rows into Bluelink, using the row_to_person function to
- transform rows to BluelinkPerson objects.
-
- `Args:`
- source: str
- String to identify that the data came from your system.
- For example, your company name.
- tbl: Table
- A parsons Table that represents people data.
- row_to_person: Callable[[dict],BluelinkPerson]
- A function that takes a dict representation of a row from the passed in tbl
- and returns a BluelinkPerson object.
-
- `Returns:`
- list[int]
- A list of https response status codes, one response for each row in the table.
- """
- people = BluelinkPerson.from_table(tbl, row_to_person)
- responses = []
- for person in people:
- response = self.upsert_person(source, person)
- responses.append(response)
- return responses
diff --git a/parsons/bluelink/person.py b/parsons/bluelink/person.py
deleted file mode 100644
index f808c090f1..0000000000
--- a/parsons/bluelink/person.py
+++ /dev/null
@@ -1,280 +0,0 @@
-import logging
-import json
-
-logger = logging.getLogger(__name__)
-
-
-class BluelinkPerson(object):
- """
- Instantiate BluelinkPerson Class.
- Used for to upserting via Bluelink connector.
- See: https://bluelinkdata.github.io/docs/BluelinkApiGuide#person-object
-
- `Args:`
- identifiers: list[BluelinkIdentifier]
- A list of BluelinkIdentifier objects.
- A BluelinkPerson must have at least 1 identifier.
- given_name: str
- First name / given name.
- family_name: str
- Last name / family name.
- phones: list[BluelinkPhone]
- A list of BluelinkPhone objects representing phone numbers.
- emails: list[BluelinkEmail]
- A list of BluelinkEmail objects representing email addresses.
- addresses: list[BluelinkAddress]
- A list of BluelinkAddress objects representing postal addresses.
- tags: list[BluelinkTag]
- Simple tags that apply to the person, eg DONOR.
- employer: str
- Name of the persons employer.
- employer_address: BluelinkAddress
- BluelinkAddress of the persons employer.
- occupation: str
- Occupation.
- scores: list[BluelinkScore]
- List of BluelinkScore objects. Scores are numeric scores, ie partisanship model.
- birthdate: str
- ISO 8601 formatted birth date: 'YYYY-MM-DD'
- details: dict
- additional custom data. must be json serializable.
- """
-
- def __init__(
- self,
- identifiers,
- given_name=None,
- family_name=None,
- phones=None,
- emails=None,
- addresses=None,
- tags=None,
- employer=None,
- employer_address=None,
- occupation=None,
- scores=None,
- birthdate=None,
- details=None,
- ):
-
- if not identifiers:
- raise Exception(
- "BluelinkPerson requires list of BluelinkIdentifiers with "
- "at least 1 BluelinkIdentifier"
- )
-
- self.identifiers = identifiers
- self.addresses = addresses
- self.emails = emails
- self.phones = phones
- self.tags = tags
- self.scores = scores
-
- self.given_name = given_name
- self.family_name = family_name
-
- self.employer = employer
- self.employer_address = employer_address
- self.occupation = occupation
- self.birthdate = birthdate
- self.details = details
-
- def __json__(self):
- """The json str representation of this BluelinkPerson object"""
- return json.dumps(self, default=lambda obj: obj.__dict__)
-
- def __eq__(self, other):
- """A quick and dirty equality check"""
- dself = json.loads(self.__json__())
- dother = json.loads(other.__json__())
- return dself == dother
-
- def __repr__(self):
- return self.__json__()
-
- @staticmethod
- def from_table(tbl, dict_to_person):
- """
- Return a list of BluelinkPerson objects from a Parsons Table.
-
- `Args:`
- tbl: Table
- A parsons Table.
- dict_to_person: Callable[[dict],BluelinkPerson]
- A function that takes a dictionary representation of a table row,
- and returns a BluelinkPerson.
- `Returns:`
- list[BluelinkPerson]
- A list of BluelinkPerson objects.
- """
- return [dict_to_person(row) for row in tbl]
-
-
-class BluelinkIdentifier(object):
- """
- Instantiate an BluelinkIdentifier object.
- BluelinkIdentifier is necessary for updating BluelinkPerson records.
-
- `Args:`
- source: str
- External system to which this ID belongs, e.g., “VAN:myCampaign”.
- Bluelink has standardized strings for source. Using these will
- allow Bluelink to correctly understand the external IDs you add.
- source (unlike identifier) is case insensitive.
- examples: BLUELINK, PDI, SALESFORCE, VAN:myCampaign, VAN:myVoters
- identifier: str
- Case-sensitive ID in the external system.
- details: dict
- dictionary of custom fields. must be serializable to json.
- """
-
- def __init__(self, source, identifier, details=None):
- self.source = source
- self.identifier = identifier
- self.details = details
-
-
-class BluelinkEmail(object):
- """
- Instantiate an BluelinkEmail object.
-
- `Args:`
- address: str
- An email address. ie "user@example.com"
- primary: bool
- True if this is known to be the primary email.
- type: str
- Type, eg: "personal", "work"
- status: str
- One of "Potential", "Subscribed", "Unsubscribed", "Bouncing", or "Spam Complaints"
- """
-
- def __init__(self, address, primary=None, type=None, status=None):
- self.address = address
- self.primary = primary
- self.type = type
- self.status = status
-
-
-class BluelinkAddress(object):
- """
- Instantiate an BluelinkAddress object.
-
- `Args`:
- address_lines: list[str]
- A list of street address lines.
- city: str
- City or other locality.
- state: str
- State in ISO 3166-2.
- postal_code: str
- Zip or other postal code.
- country: str
- ISO 3166-1 Alpha-2 country code.
- type: str
- The type. ie: "home", "mailing".
- venue: str
- The venue name, if relevant.
- status: str
- A value representing the status of the address. "Potential", "Verified" or "Bad"
- """
-
- def __init__(
- self,
- address_lines=None,
- city=None,
- state=None,
- postal_code=None,
- country=None,
- type=None,
- venue=None,
- status=None,
- ):
-
- self.address_lines = address_lines or []
- self.city = city
- self.state = state
- self.postal_code = postal_code
- self.country = country
-
- self.type = type
- self.venue = venue
- self.status = status
-
-
-class BluelinkPhone(object):
- """
- Instantiate a BluelinkPhone object.
-
- `Args:`
- number: str
- A phone number. May or may not include country code.
- primary: bool
- True if this is known to be the primary phone.
- description: str
- Free for description.
- type: str
- Type, eg: "Home", "Work", "Mobile"
- country: str
- ISO 3166-1 Alpha-2 country code.
- sms_capable: bool
- True if this number can accept SMS.
- do_not_call: bool
- True if this number is on the US FCC Do Not Call Registry.
- details: dict
- Additional data dictionary. Must be json serializable.
- """
-
- def __init__(
- self,
- number,
- primary=None,
- description=None,
- type=None,
- country=None,
- sms_capable=None,
- do_not_call=None,
- details=None,
- ):
- self.number = number
- self.primary = primary
- self.description = description
- self.type = type
- self.country = country
- self.sms_capable = sms_capable
- self.do_not_call = do_not_call
- self.details = details
-
-
-class BluelinkTag(object):
- """
- Instantiate a BluelinkTag object.
-
- `Args:`
- tag: str
- A tag string; convention is either a simple string
- or a string with a prefix separated by a colon, e.g., “DONOR:GRASSROOTS”
- """
-
- def __init__(self, tag):
- self.tag = tag
-
-
-class BluelinkScore(object):
- """
- Instantiate a score object.
- Represents some kind of numeric score.
-
- `Args`:
- score: float
- Numeric score.
- score_type: str
- Type, eg: "Partisanship model".
- source: str
- Original source of this score.
- """
-
- def __init__(self, score, score_type, source):
- self.score = score
- self.score_type = score_type
- self.source = source
diff --git a/parsons/google/google_bigquery.py b/parsons/google/google_bigquery.py
index fecb15ed2f..9b2f6330c5 100644
--- a/parsons/google/google_bigquery.py
+++ b/parsons/google/google_bigquery.py
@@ -11,6 +11,7 @@
from google.cloud import bigquery, exceptions
from google.cloud.bigquery import dbapi
from google.cloud.bigquery.job import LoadJobConfig
+from google.oauth2.credentials import Credentials
from parsons.databases.database_connector import DatabaseConnector
from parsons.databases.table import BaseTable
@@ -143,7 +144,7 @@ class GoogleBigQuery(DatabaseConnector):
def __init__(
self,
- app_creds=None,
+ app_creds: Optional[Union[str, dict, Credentials]] = None,
project=None,
location=None,
client_options: dict = {
@@ -156,7 +157,11 @@ def __init__(
):
self.app_creds = app_creds
- setup_google_application_credentials(app_creds)
+ if isinstance(app_creds, Credentials):
+ self.credentials = app_creds
+ else:
+ self.credentials = None
+ setup_google_application_credentials(app_creds)
self.project = project
self.location = location
@@ -185,6 +190,7 @@ def client(self):
project=self.project,
location=self.location,
client_options=self.client_options,
+ credentials=self.credentials,
)
return self._client
diff --git a/parsons/google/google_sheets.py b/parsons/google/google_sheets.py
index 67deaefc1c..1ab1b2dd5b 100644
--- a/parsons/google/google_sheets.py
+++ b/parsons/google/google_sheets.py
@@ -3,7 +3,7 @@
import logging
from parsons.etl.table import Table
-from parsons.google.utitities import setup_google_application_credentials
+from parsons.google.utitities import setup_google_application_credentials, hexavigesimal
import gspread
from google.oauth2.service_account import Credentials
@@ -293,6 +293,63 @@ def append_to_sheet(
sheet.update_cells(cells, value_input_option=value_input_option)
logger.info(f"Appended {table.num_rows} rows to worksheet.")
+ def paste_data_in_sheet(
+ self, spreadsheet_id, table, worksheet=0, header=True, startrow=0, startcol=0
+ ):
+ """
+ Pastes data from a Parsons table to a Google sheet. Note that this may overwrite
+ presently existing data. This function is useful for adding data to a subsection
+ if an existint sheet that will have other existing data - constrast to
+ `overwrite_sheet` (which will fully replace any existing data) and `append_to_sheet`
+ (whuch sticks the data only after all other existing data).
+
+ `Args:`
+ spreadsheet_id: str
+ The ID of the spreadsheet (Tip: Get this from the spreadsheet URL).
+ table: obj
+ Parsons table
+ worksheet: str or int
+ The index or the title of the worksheet. The index begins with 0.
+ header: bool
+ Whether or not the header row gets pasted with the data.
+ startrow: int
+ Starting row position of pasted data. Counts from 0.
+ startcol: int
+ Starting column position of pasted data. Counts from 0.
+ """
+ sheet = self._get_worksheet(spreadsheet_id, worksheet)
+
+ number_of_columns = len(table.columns)
+ number_of_rows = table.num_rows + 1 if header else table.num_rows
+
+ if not number_of_rows or not number_of_columns: # No data to paste
+ logger.warning(
+ f"No data available to paste, table size "
+ f"({number_of_rows}, {number_of_columns}). Skipping."
+ )
+ return
+
+ # gspread uses ranges like "C3:J7", so we need to convert to this format
+ data_range = (
+ hexavigesimal(startcol + 1)
+ + str(startrow + 1)
+ + ":"
+ + hexavigesimal(startcol + number_of_columns)
+ + str(startrow + number_of_rows)
+ )
+
+ # Unpack data. Hopefully this is small enough for memory
+ data = [[]] * table.num_rows
+ for row_num, row in enumerate(table.data):
+ data[row_num] = list(row)
+
+ if header:
+ sheet.update(data_range, [table.columns] + data)
+ else:
+ sheet.update(data_range, data)
+
+ logger.info(f"Pasted data to {data_range} in worksheet.")
+
def overwrite_sheet(
self, spreadsheet_id, table, worksheet=0, user_entered_value=False, **kwargs
):
diff --git a/parsons/google/utitities.py b/parsons/google/utitities.py
index abda0bf212..75cdf01de0 100644
--- a/parsons/google/utitities.py
+++ b/parsons/google/utitities.py
@@ -22,3 +22,29 @@ def setup_google_application_credentials(
creds_path = credentials
os.environ[env_var_name] = creds_path
+
+
+def hexavigesimal(n: int) -> str:
+ """
+ Converts an integer value to the type of strings you see on spreadsheets
+ (A, B,...,Z, AA, AB, ...).
+
+ Code based on
+ https://stackoverflow.com/questions/16190452/converting-from-number-to-hexavigesimal-letters
+
+ `Args:`
+ n: int
+ A positive valued integer.
+
+ `Returns:`
+ str
+ The hexavigeseimal representation of n
+ """
+ if n < 1:
+ raise ValueError(f"This function only works for positive integers. Provided value {n}.")
+
+ chars = ""
+ while n != 0:
+ chars = chr((n - 1) % 26 + 65) + chars # 65 makes us start at A
+ n = (n - 1) // 26
+ return chars
diff --git a/parsons/ngpvan/email.py b/parsons/ngpvan/email.py
new file mode 100644
index 0000000000..1e5c55c079
--- /dev/null
+++ b/parsons/ngpvan/email.py
@@ -0,0 +1,150 @@
+from parsons.etl.table import Table
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class Email(object):
+ """
+ Instantiate the Email class.
+
+ You can find the docs for the NGP VAN Email API here:
+ https://docs.ngpvan.com/reference/email-overview
+ """
+
+ def __init__(self, van_connection):
+
+ self.connection = van_connection
+
+ def get_emails(self, ascending: bool = True) -> Table:
+ """
+ Get emails.
+
+ `Args:`
+ ascending : Bool
+ sorts results in ascending or descending order
+ for the dateModified field. Defaults to True (ascending).
+
+ `Returns:`
+ Parsons Table
+ See :ref:`parsons-table` for output options.
+ """
+ if ascending:
+ params = {
+ "$orderby": "dateModified asc",
+ }
+ if not ascending:
+ params = {
+ "$orderby": "dateModified desc",
+ }
+
+ tbl = Table(self.connection.get_request("email/messages", params=params))
+ logger.debug(f"Found {tbl.num_rows} emails.")
+ return tbl
+
+ def get_email(self, email_id: int, expand: bool = True) -> Table:
+ """
+ Get an email.
+
+ Note that it takes some time for the system to aggregate opens and click-throughs,
+ so data can be delayed up to 15 minutes.
+
+ `Args:`
+ email_id : int
+ The email id.
+ expand : bool
+ Optional; expands the email message to include the email content and
+ statistics. Defaults to True.
+
+ `Returns:`
+ dict
+ """
+
+ params = {
+ "$expand": (
+ "emailMessageContent, EmailMessageContentDistributions"
+ if expand
+ else None
+ ),
+ }
+
+ r = self.connection.get_request(f"email/message/{email_id}", params=params)
+ logger.debug(f"Found email {email_id}.")
+ return r
+
+ def get_email_stats(self) -> Table:
+ """
+ Get stats for all emails, aggregating any A/B tests.
+
+ `Args:`
+ emails : list
+ A list of email message details.
+
+ `Returns:`
+ Parsons Table
+ See :ref:`parsons-table` for output options.
+ """
+
+ email_list = []
+
+ final_email_list = []
+
+ emails = self.get_emails()
+
+ foreign_message_ids = [email["foreignMessageId"] for email in emails]
+
+ for fmid in foreign_message_ids:
+ email = self.get_email(fmid)
+ email_list.append(email)
+
+ for email in email_list:
+ d = {}
+ d["name"] = email["name"]
+ d["createdBy"] = email["createdBy"]
+ d["dateCreated"] = email["dateCreated"]
+ d["dateModified"] = email["dateModified"]
+ d["dateScheduled"] = email["dateScheduled"]
+ d["foreignMessageId"] = email["foreignMessageId"]
+ d["recipientCount"] = 0
+ d["bounceCount"] = 0
+ d["contributionCount"] = 0
+ d["contributionTotal"] = 0
+ d["formSubmissionCount"] = 0
+ d["linksClickedCount"] = 0
+ d["machineOpenCount"] = 0
+ d["openCount"] = 0
+ d["unsubscribeCount"] = 0
+ try:
+ for i in email["emailMessageContent"]:
+ d["recipientCount"] += i["emailMessageContentDistributions"][
+ "recipientCount"
+ ]
+ d["bounceCount"] += i["emailMessageContentDistributions"][
+ "bounceCount"
+ ]
+ d["contributionCount"] += i["emailMessageContentDistributions"][
+ "contributionCount"
+ ]
+ d["contributionTotal"] += i["emailMessageContentDistributions"][
+ "contributionTotal"
+ ]
+ d["formSubmissionCount"] += i["emailMessageContentDistributions"][
+ "formSubmissionCount"
+ ]
+ d["linksClickedCount"] += i["emailMessageContentDistributions"][
+ "linksClickedCount"
+ ]
+ d["machineOpenCount"] += i["emailMessageContentDistributions"][
+ "machineOpenCount"
+ ]
+ d["openCount"] += i["emailMessageContentDistributions"]["openCount"]
+ d["unsubscribeCount"] += i["emailMessageContentDistributions"][
+ "unsubscribeCount"
+ ]
+ except TypeError as e:
+ logger.info(str(e))
+ pass
+
+ final_email_list.append(d)
+
+ return Table(final_email_list)
diff --git a/parsons/ngpvan/van.py b/parsons/ngpvan/van.py
index c57bf835cb..e5f6897ce5 100644
--- a/parsons/ngpvan/van.py
+++ b/parsons/ngpvan/van.py
@@ -1,5 +1,6 @@
import logging
from parsons.ngpvan.events import Events
+from parsons.ngpvan.email import Email
from parsons.ngpvan.van_connector import VANConnector
from parsons.ngpvan.people import People
from parsons.ngpvan.saved_lists import SavedLists, Folders, ExportJobs
@@ -24,6 +25,7 @@
class VAN(
People,
Events,
+ Email,
SavedLists,
PrintedLists,
Folders,
diff --git a/requirements-dev.txt b/requirements-dev.txt
index 0bee9cdfa2..4ef91afbcc 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,5 +1,5 @@
# Testing Requirements
-black==24.2.0
+black==24.3.0
Flake8-pyproject==1.2.3
flake8==7.0.0
pytest-datadir==1.5.0
diff --git a/requirements.txt b/requirements.txt
index 43aa054ce4..d911a8f734 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -24,7 +24,7 @@ newmode==0.1.6
oauth2client==4.1.3
paramiko==3.4.0
petl==1.7.15
-psycopg2-binary==2.9.3
+psycopg2-binary==2.9.9
PyGitHub==1.51
PyJWT==2.4.0 # Otherwise `import jwt` would refer to python-jwt package
python-dateutil==2.8.2
@@ -33,7 +33,7 @@ requests_oauthlib==1.3.0
simple-salesforce==1.11.6
simplejson==3.16.0
slackclient==1.3.0
-SQLAlchemy==1.3.23
+sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0 # Prefect does not work with 1.4.33 and >3.0.0 has breaking changes
suds-py3==1.3.4.0
surveygizmo==1.2.3
twilio==8.2.1
@@ -45,4 +45,4 @@ xmltodict==0.11.0
# TODO Remove when we have a TMC-specific Docker image
jinja2>=3.0.2
selenium==3.141.0
-us==3.1.1
\ No newline at end of file
+us==3.1.1
diff --git a/setup.py b/setup.py
index 04655af48e..18451928eb 100644
--- a/setup.py
+++ b/setup.py
@@ -40,12 +40,15 @@ def main():
"oauth2client",
"validate-email",
],
- "mysql": ["mysql-connector-python", "SQLAlchemy"],
+ "mysql": [
+ "mysql-connector-python",
+ "sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0",
+ ],
"newmode": ["newmode"],
"ngpvan": ["suds-py3"],
"mobilecommons": ["bs4"],
- "postgres": ["psycopg2-binary", "SQLAlchemy"],
- "redshift": ["boto3", "psycopg2-binary", "SQLAlchemy"],
+ "postgres": ["psycopg2-binary>=2.9.9", "sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0",],
+ "redshift": ["boto3", "psycopg2-binary>=2.9.9", "sqlalchemy >= 1.4.22, != 1.4.33, < 2.0.0",],
"s3": ["boto3"],
"salesforce": ["simple-salesforce"],
"sftp": ["paramiko"],
@@ -89,7 +92,7 @@ def main():
],
python_requires=">=3.7.0,<3.12.0",
long_description=long_description,
- long_description_content_type='text/markdown'
+ long_description_content_type="text/markdown",
)
diff --git a/test/test_action_kit.py b/test/test_action_kit.py
index 378d6c55f3..6a0b3fb454 100644
--- a/test/test_action_kit.py
+++ b/test/test_action_kit.py
@@ -89,12 +89,14 @@ def test_update_user(self):
type(resp_mock.patch()).status_code = mock.PropertyMock(return_value=202)
self.actionkit.conn = resp_mock
- self.actionkit.update_user(123, last_name="new name")
+ res = self.actionkit.update_user(123, last_name="new name")
self.actionkit.conn.patch.assert_called_with(
"https://domain.actionkit.com/rest/v1/user/123/",
data=json.dumps({"last_name": "new name"}),
)
+ assert res.status_code == 202
+
def test_update_event(self):
# Test update event
diff --git a/test/test_bluelink/test_bluelink.py b/test/test_bluelink/test_bluelink.py
deleted file mode 100644
index 5c15ea5f70..0000000000
--- a/test/test_bluelink/test_bluelink.py
+++ /dev/null
@@ -1,117 +0,0 @@
-import unittest
-import requests_mock
-from parsons import Table, Bluelink
-from parsons.bluelink import BluelinkPerson, BluelinkIdentifier, BluelinkEmail
-
-
-class TestBluelink(unittest.TestCase):
- @requests_mock.Mocker()
- def setUp(self, m):
- self.bluelink = Bluelink("fake_user", "fake_password")
-
- @staticmethod
- def row_to_person(row):
- """
- dict -> BluelinkPerson
- Transforms a parsons Table row to a BluelinkPerson.
- This function is passed into bulk_upsert_person along with a Table
- """
- email = row["email"]
- return BluelinkPerson(
- identifiers=[
- BluelinkIdentifier(source="FAKESOURCE", identifier=email),
- ],
- emails=[BluelinkEmail(address=email, primary=True)],
- family_name=row["family_name"],
- given_name=row["given_name"],
- )
-
- @staticmethod
- def get_table():
- return Table(
- [
- {
- "given_name": "Bart",
- "family_name": "Simpson",
- "email": "bart@springfield.net",
- },
- {
- "given_name": "Homer",
- "family_name": "Simpson",
- "email": "homer@springfield.net",
- },
- ]
- )
-
- @requests_mock.Mocker()
- def test_bulk_upsert_person(self, m):
- """
- This function demonstrates how to use a "row_to_person" function to bulk
- insert people using a Table as the data source
- """
- # Mock POST requests to api
- m.post(self.bluelink.api_url)
-
- # get data as a parsons Table
- tbl = self.get_table()
-
- # String to identify that the data came from your system. For example, your company name.
- source = "BLUELINK-PARSONS-TEST"
-
- # call bulk_upsert_person
- # passing in the source, the Table, and the function that maps a Table row -> BluelinkPerson
- self.bluelink.bulk_upsert_person(source, tbl, self.row_to_person)
-
- @requests_mock.Mocker()
- def test_upsert_person(self, m):
- """
- This function demonstrates how to insert a single BluelinkPerson record
- """
- # Mock POST requests to api
- m.post(self.bluelink.api_url)
-
- # create a BluelinkPerson object
- # The BluelinkIdentifier is pretending that the user can be
- # identified in SALESFORCE with FAKE_ID as her id
- person = BluelinkPerson(
- identifiers=[BluelinkIdentifier(source="SALESFORCE", identifier="FAKE_ID")],
- given_name="Jane",
- family_name="Doe",
- emails=[BluelinkEmail(address="jdoe@example.com", primary=True)],
- )
-
- # String to identify that the data came from your system. For example, your company name.
- source = "BLUELINK-PARSONS-TEST"
-
- # call upsert_person
- self.bluelink.upsert_person(source, person)
-
- def test_table_to_people(self):
- """
- Test transforming a parsons Table -> list[BluelinkPerson]
- """
- # setup
- tbl = self.get_table()
-
- # function under test
- actual_people = BluelinkPerson.from_table(tbl, self.row_to_person)
-
- # expected:
- person1 = BluelinkPerson(
- identifiers=[
- BluelinkIdentifier(source="FAKESOURCE", identifier="bart@springfield.net")
- ],
- emails=[BluelinkEmail(address="bart@springfield.net", primary=True)],
- family_name="Simpson",
- given_name="Bart",
- )
- person2 = BluelinkPerson(
- identifiers=[
- BluelinkIdentifier(source="FAKESOURCE", identifier="homer@springfield.net")
- ],
- emails=[BluelinkEmail(address="homer@springfield.net", primary=True)],
- family_name="Simpson",
- given_name="Homer",
- )
- expected_people = [person1, person2]
- self.assertEqual(actual_people, expected_people)
diff --git a/test/test_google/test_google_sheets.py b/test/test_google/test_google_sheets.py
index d7672b76a7..ed6811bb6a 100644
--- a/test/test_google/test_google_sheets.py
+++ b/test/test_google/test_google_sheets.py
@@ -123,6 +123,82 @@ def test_append_user_entered_to_spreadsheet(self):
self.assertEqual(formula_vals[0], "27")
self.assertEqual(formula_vals[1], "Budapest")
+ def test_paste_data_in_sheet(self):
+ # Testing if we can paste data to a spreadsheet
+ # TODO: there's probably a smarter way to test this code
+ self.google_sheets.add_sheet(self.spreadsheet_id, "PasteDataSheet")
+
+ paste_table1 = Table(
+ [
+ {"col1": 1, "col2": 2},
+ {"col1": 5, "col2": 6},
+ ]
+ )
+ paste_table2 = Table(
+ [
+ {"col3": 3, "col4": 4},
+ {"col3": 7, "col4": 8},
+ ]
+ )
+ paste_table3 = Table(
+ [
+ {"col1": 9, "col2": 10},
+ {"col1": 13, "col2": 14},
+ ]
+ )
+ paste_table4 = Table(
+ [
+ {"col3": 11, "col4": 12},
+ {"col3": 15, "col4": 16},
+ ]
+ )
+
+ # When we read the spreadsheet, it assumes data is all strings
+ expected_table = Table(
+ [
+ {"col1": "1", "col2": "2", "col3": "3", "col4": "4"},
+ {"col1": "5", "col2": "6", "col3": "7", "col4": "8"},
+ {"col1": "9", "col2": "10", "col3": "11", "col4": "12"},
+ {"col1": "13", "col2": "14", "col3": "15", "col4": "16"},
+ ]
+ )
+
+ self.google_sheets.paste_data_in_sheet(
+ self.spreadsheet_id,
+ paste_table1,
+ worksheet="PasteDataSheet",
+ header=True,
+ startrow=0,
+ startcol=0,
+ )
+ self.google_sheets.paste_data_in_sheet(
+ self.spreadsheet_id,
+ paste_table2,
+ worksheet="PasteDataSheet",
+ header=True,
+ startrow=0,
+ startcol=2,
+ )
+ self.google_sheets.paste_data_in_sheet(
+ self.spreadsheet_id,
+ paste_table3,
+ worksheet="PasteDataSheet",
+ header=False,
+ startrow=3,
+ startcol=0,
+ )
+ self.google_sheets.paste_data_in_sheet(
+ self.spreadsheet_id,
+ paste_table4,
+ worksheet="PasteDataSheet",
+ header=False,
+ startrow=3,
+ startcol=2,
+ )
+
+ result_table = self.google_sheets.get_worksheet(self.spreadsheet_id, "PasteDataSheet")
+ self.assertEqual(result_table.to_dicts(), expected_table.to_dicts())
+
def test_overwrite_spreadsheet(self):
new_table = Table(
[
diff --git a/test/test_google/test_utilities.py b/test/test_google/test_utilities.py
index 5a6b8c2966..accb506f90 100644
--- a/test/test_google/test_utilities.py
+++ b/test/test_google/test_utilities.py
@@ -72,3 +72,16 @@ def test_credentials_are_valid_after_double_call(self):
actual = fsnd.read()
self.assertEqual(self.cred_contents, json.loads(actual))
self.assertEqual(ffst.read(), actual)
+
+
+class TestHexavigesimal(unittest.TestCase):
+
+ def test_returns_A_on_1(self):
+ self.assertEqual(util.hexavigesimal(1), "A")
+
+ def test_returns_AA_on_27(self):
+ self.assertEqual(util.hexavigesimal(27), "AA")
+
+ def test_returns_error_on_0(self):
+ with self.assertRaises(ValueError):
+ util.hexavigesimal(0)
diff --git a/test/test_van/test_email.py b/test/test_van/test_email.py
new file mode 100644
index 0000000000..faf9aabcee
--- /dev/null
+++ b/test/test_van/test_email.py
@@ -0,0 +1,111 @@
+import unittest
+import os
+import requests_mock
+from parsons import VAN, Table
+
+
+def assert_matching_tables(table1, table2, ignore_headers=False):
+ if ignore_headers:
+ data1 = table1.data
+ data2 = table2.data
+ else:
+ data1 = table1
+ data2 = table2
+
+ if isinstance(data1, Table) and isinstance(data2, Table):
+ assert data1.num_rows == data2.num_rows
+
+ for r1, r2 in zip(data1, data2):
+ # Cast both rows to lists, in case they are different types of collections. Must call
+ # .items() on dicts to compare content of collections
+ if isinstance(r1, dict):
+ r1 = r1.items()
+ if isinstance(r2, dict):
+ r2 = r2.items()
+
+ assert list(r1) == list(r2)
+
+
+os.environ["VAN_API_KEY"] = "SOME_KEY"
+
+mock_response = [
+ {
+ "foreignMessageId": "oK2ahdAcEe6F-QAiSCI3lA2",
+ "name": "Test Email",
+ "createdBy": "Joe Biden",
+ "dateCreated": "2024-02-20T13:20:00Z",
+ "dateScheduled": "2024-02-20T15:55:00Z",
+ "campaignID": 0,
+ "dateModified": "2024-02-20T15:54:36.27Z",
+ "emailMessageContent": None,
+ },
+ {
+ "foreignMessageId": "rjzc2szzEe6F-QAiSCI3lA2",
+ "name": "Test Email 2",
+ "createdBy": "Joe Biden",
+ "dateCreated": "2024-02-16T12:49:00Z",
+ "dateScheduled": "2024-02-16T13:29:00Z",
+ "campaignID": 0,
+ "dateModified": "2024-02-16T13:29:16.453Z",
+ "emailMessageContent": None,
+ },
+ {
+ "foreignMessageId": "_E1AfcnkEe6F-QAiSCI3lA2",
+ "name": "Test Email 3",
+ "createdBy": "Joe Biden",
+ "dateCreated": "2024-02-12T15:26:00Z",
+ "dateScheduled": "2024-02-13T11:22:00Z",
+ "campaignID": 0,
+ "dateModified": "2024-02-13T11:22:28.273Z",
+ "emailMessageContent": None,
+ },
+ {
+ "foreignMessageId": "6GTLBsUwEe62YAAiSCIxlw2",
+ "name": "Test Email 4",
+ "createdBy": "Joe Biden",
+ "dateCreated": "2024-02-06T15:47:00Z",
+ "dateScheduled": "2024-02-07T10:32:00Z",
+ "campaignID": 0,
+ "dateModified": "2024-02-07T10:31:55.16Z",
+ "emailMessageContent": None,
+ },
+ {
+ "foreignMessageId": "mgTdmcEiEe62YAAiSCIxlw2",
+ "name": "Test Email 5",
+ "createdBy": "Joe Biden",
+ "dateCreated": "2024-02-01T11:55:00Z",
+ "dateScheduled": "2024-02-01T16:08:00Z",
+ "campaignID": 0,
+ "dateModified": "2024-02-01T16:08:10.737Z",
+ "emailMessageContent": None,
+ },
+]
+
+
+class TestEmail(unittest.TestCase):
+ def setUp(self):
+ self.van = VAN(os.environ["VAN_API_KEY"], db="MyVoters", raise_for_status=False)
+
+ @requests_mock.Mocker()
+ def test_get_email_messages(self, m):
+ m.get(
+ self.van.connection.uri + "email/messages",
+ json=mock_response,
+ status_code=200,
+ )
+
+ response = self.van.get_emails()
+
+ assert_matching_tables(response, mock_response)
+
+ @requests_mock.Mocker()
+ def test_get_email_message(self, m):
+ m.get(
+ self.van.connection.uri + "email/message/1",
+ json=mock_response[0],
+ status_code=200,
+ )
+
+ response = self.van.get_email(1)
+
+ assert_matching_tables(response, mock_response[0])