diff --git a/docs/index.rst b/docs/index.rst index 0ce117ddac..2183d89469 100755 --- a/docs/index.rst +++ b/docs/index.rst @@ -205,6 +205,7 @@ Indices and tables hustle mailchimp mobilize_america + nation_builder newmode ngpvan p2a diff --git a/docs/nation_builder.rst b/docs/nation_builder.rst new file mode 100644 index 0000000000..649f52dace --- /dev/null +++ b/docs/nation_builder.rst @@ -0,0 +1,52 @@ +NationBuilder +============== + +******** +Overview +******** + +The NationBuilder class allows you to interact with the NationBuilder API. Users of this Parsons integration can download a full list of people, update and upsert people. + +.. note:: + Authentication + In order to use this class you need your nation slug and access token. To get your access token login to your nation and navigate to ``Settings > Developer > API Token`` and create a new token. You can get more info in the `NationBuilder API docs `_. + +========== +Quickstart +========== + +To instantiate the NationBuilder class, you can either store your ``NB_SLUG`` and ``NB_ACCESS_TOKEN`` keys as environment +variables or pass them in as arguments: + +.. code-block:: python + + from parsons import NationBuilder + + # First approach: Use API key environment variables + + # In bash, set your environment variables like so: + # export NB_SLUG='my-nation-slug' + # export NB_ACCESS_TOKEN='MY_ACCESS_TOKEN' + nb = NationBuilder() + + # Second approach: Pass API keys as arguments + nb = NationBuilder(slug='my-nation-slug', access_token='MY_ACCESS_TOKEN') + +You can then make a request to get all people and save its data to a Parsons table using the method, ``get_people()``: + +.. code-block:: python + + # Create Parsons table with people data from API + parsons_table = nb.get_people() + + # Save people as CSV + parsons_table.to_csv('people.csv') + +The above example shows how to create a Parsons table with all people registered in your NationBuilder nation. + +*** +API +*** + +.. autoclass :: parsons.NationBuilder + :inherited-members: diff --git a/parsons/__init__.py b/parsons/__init__.py index bd30b0fff9..eff0c761cf 100644 --- a/parsons/__init__.py +++ b/parsons/__init__.py @@ -1,7 +1,7 @@ # Provide shortcuts to importing Parsons submodules and set up logging import importlib -import os import logging +import os from parsons.etl.table import Table @@ -63,6 +63,7 @@ ("parsons.hustle.hustle", "Hustle"), ("parsons.mailchimp.mailchimp", "Mailchimp"), ("parsons.mobilize_america.ma", "MobilizeAmerica"), + ("parsons.nation_builder.nation_builder", "NationBuilder"), ("parsons.newmode.newmode", "Newmode"), ("parsons.ngpvan.van", "VAN"), ("parsons.notifications.gmail", "Gmail"), diff --git a/parsons/nation_builder/__init__.py b/parsons/nation_builder/__init__.py new file mode 100644 index 0000000000..3520433033 --- /dev/null +++ b/parsons/nation_builder/__init__.py @@ -0,0 +1,3 @@ +from parsons.nation_builder.nation_builder import NationBuilder + +__all__ = ["NationBuilder"] diff --git a/parsons/nation_builder/nation_builder.py b/parsons/nation_builder/nation_builder.py new file mode 100644 index 0000000000..6a6ff6a920 --- /dev/null +++ b/parsons/nation_builder/nation_builder.py @@ -0,0 +1,221 @@ +import json +import logging +import time +from typing import Any, Dict, Optional, Tuple, cast +from urllib.parse import parse_qs, urlparse + +from parsons import Table +from parsons.utilities import check_env +from parsons.utilities.api_connector import APIConnector + +logger = logging.getLogger(__name__) + + +class NationBuilder: + """ + Instantiate the NationBuilder class + + `Args:` + slug: str + The Nation Builder slug Not required if ``NB_SLUG`` env variable set. The slug is the + nation slug of the nation from which your application is requesting approval to retrieve + data via the NationBuilder API. For example, your application's user could provide this + slug via a text field in your application. + access_token: str + The Nation Builder access_token Not required if ``NB_ACCESS_TOKEN`` env variable set. + """ + + def __init__( + self, slug: Optional[str] = None, access_token: Optional[str] = None + ) -> None: + slug = check_env.check("NB_SLUG", slug) + token = check_env.check("NB_ACCESS_TOKEN", access_token) + + headers = {"Content-Type": "application/json", "Accept": "application/json"} + headers.update(NationBuilder.get_auth_headers(token)) + + self.client = APIConnector(NationBuilder.get_uri(slug), headers=headers) + + @classmethod + def get_uri(cls, slug: Optional[str]) -> str: + if slug is None: + raise ValueError("slug can't None") + + if not isinstance(slug, str): + raise ValueError("slug must be an str") + + if len(slug.strip()) == 0: + raise ValueError("slug can't be an empty str") + + return f"https://{slug}.nationbuilder.com/api/v1" + + @classmethod + def get_auth_headers(cls, access_token: Optional[str]) -> Dict[str, str]: + if access_token is None: + raise ValueError("access_token can't None") + + if not isinstance(access_token, str): + raise ValueError("access_token must be an str") + + if len(access_token.strip()) == 0: + raise ValueError("access_token can't be an empty str") + + return {"authorization": f"Bearer {access_token}"} + + @classmethod + def parse_next_params(cls, next_value: str) -> Tuple[str, str]: + next_params = parse_qs(urlparse(next_value).query) + + if "__nonce" not in next_params: + raise ValueError("__nonce param not found") + + if "__token" not in next_params: + raise ValueError("__token param not found") + + nonce = next_params["__nonce"][0] + token = next_params["__token"][0] + + return nonce, token + + @classmethod + def make_next_url(cls, original_url: str, nonce: str, token: str) -> str: + return f"{original_url}?limit=100&__nonce={nonce}&__token={token}" + + def get_people(self) -> Table: + """ + `Returns:` + A Table of all people stored in Nation Builder. + """ + data = [] + original_url = "people" + + url = f"{original_url}" + + while True: + try: + logging.debug("sending request %s" % url) + response = self.client.get_request(url) + + res = response.get("results", None) + + if res is None: + break + + logging.debug("response got %s records" % len(res)) + + data.extend(res) + + if response.get("next", None): + nonce, token = NationBuilder.parse_next_params(response["next"]) + url = NationBuilder.make_next_url(original_url, nonce, token) + else: + break + except Exception as error: + logging.error("error requesting data from Nation Builder: %s" % error) + + wait_time = 30 + logging.info("waiting %d seconds before retrying" % wait_time) + time.sleep(wait_time) + + return Table(data) + + def update_person(self, person_id: str, person: Dict[str, Any]) -> dict[str, Any]: + """ + This method updates a person with the provided id to have the provided data. It returns a + full representation of the updated person. + + `Args:` + person_id: str + Nation Builder person id. + data: dict + Nation builder person object. + For example {"email": "user@example.com", "tags": ["foo", "bar"]} + Docs: https://nationbuilder.com/people_api + `Returns:` + A person object with the updated data. + """ + if person_id is None: + raise ValueError("person_id can't None") + + if not isinstance(person_id, str): + raise ValueError("person_id must be a str") + + if len(person_id.strip()) == 0: + raise ValueError("person_id can't be an empty str") + + if not isinstance(person, dict): + raise ValueError("person must be a dict") + + url = f"people/{person_id}" + response = self.client.put_request(url, data=json.dumps({"person": person})) + response = cast(dict[str, Any], response) + + return response + + def upsert_person( + self, person: Dict[str, Any] + ) -> Tuple[bool, dict[str, Any] | None]: + """ + Updates a matched person or creates a new one if the person doesn't exist. + + This method attempts to match the input person resource to a person already in the + nation. If a match is found, the matched person is updated. If a match is not found, a new + person is created. Matches are found by including one of the following IDs in the request: + + - civicrm_id + - county_file_id + - dw_id + - external_id + - email + - facebook_username + - ngp_id + - salesforce_id + - twitter_login + - van_id + + `Args:` + data: dict + Nation builder person object. + For example {"email": "user@example.com", "tags": ["foo", "bar"]} + Docs: https://nationbuilder.com/people_api + `Returns:` + A tuple of `created` and `person` object with the updated data. If the request fails + the method will return a tuple of `False` and `None`. + """ + + _required_keys = [ + "civicrm_id", + "county_file_id", + "dw_id", + "external_id", + "email", + "facebook_username", + "ngp_id", + "salesforce_id", + "twitter_login", + "van_id", + ] + + if not isinstance(person, dict): + raise ValueError("person must be a dict") + + has_required_key = any(x in person for x in _required_keys) + + if not has_required_key: + _keys = ", ".join(_required_keys) + raise ValueError(f"person dict must contain at least one key of {_keys}") + + url = "people/push" + response = self.client.request(url, "PUT", data=json.dumps({"person": person})) + + self.client.validate_response(response) + + if response.status_code == 200: + if self.client.json_check(response): + return (False, response.json()) + + if response.status_code == 201: + if self.client.json_check(response): + return (True, response.json()) + + return (False, None) diff --git a/test/test_nation_builder/__init__.py b/test/test_nation_builder/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/test/test_nation_builder/fixtures.py b/test/test_nation_builder/fixtures.py new file mode 100644 index 0000000000..fb5a0eaee6 --- /dev/null +++ b/test/test_nation_builder/fixtures.py @@ -0,0 +1,350 @@ +GET_PEOPLE_RESPONSE = { + "results": [ + { + "birthdate": None, + "city_district": None, + "civicrm_id": None, + "county_district": None, + "county_file_id": None, + "created_at": "2023-06-22T11:41:56-07:00", + "datatrust_id": None, + "do_not_call": False, + "do_not_contact": False, + "dw_id": None, + "email": "foo@example.com", + "email_opt_in": True, + "employer": None, + "external_id": None, + "federal_district": None, + "fire_district": None, + "first_name": "Foo", + "has_facebook": False, + "id": 4, + "is_twitter_follower": False, + "is_volunteer": False, + "judicial_district": None, + "labour_region": None, + "last_name": "Bar", + "linkedin_id": None, + "mobile": None, + "mobile_opt_in": None, + "middle_name": "", + "nbec_guid": None, + "ngp_id": None, + "note": None, + "occupation": None, + "party": None, + "pf_strat_id": None, + "phone": None, + "precinct_id": None, + "primary_address": None, + "profile_image_url_ssl": "https://example.com/assets/notifier/profile-avatar.png", + "recruiter_id": None, + "rnc_id": None, + "rnc_regid": None, + "salesforce_id": None, + "school_district": None, + "school_sub_district": None, + "sex": None, + "signup_type": 0, + "state_file_id": None, + "state_lower_district": None, + "state_upper_district": None, + "support_level": None, + "supranational_district": None, + "tags": ["zoot", "boot"], + "twitter_id": None, + "twitter_name": None, + "updated_at": "2023-06-22T11:41:56-07:00", + "van_id": None, + "village_district": None, + "ward": None, + "work_phone_number": None, + }, + { + "birthdate": None, + "city_district": None, + "civicrm_id": None, + "county_district": None, + "county_file_id": None, + "created_at": "2023-06-22T08:21:00-07:00", + "datatrust_id": None, + "do_not_call": False, + "do_not_contact": False, + "dw_id": None, + "email": "bar@example.com", + "email_opt_in": True, + "employer": None, + "external_id": None, + "federal_district": None, + "fire_district": None, + "first_name": "Zoo", + "has_facebook": False, + "id": 2, + "is_twitter_follower": False, + "is_volunteer": False, + "judicial_district": None, + "labour_region": None, + "last_name": "Baz", + "linkedin_id": None, + "mobile": None, + "mobile_opt_in": True, + "middle_name": "", + "nbec_guid": None, + "ngp_id": None, + "note": None, + "occupation": None, + "party": None, + "pf_strat_id": None, + "phone": None, + "precinct_id": None, + "primary_address": None, + "profile_image_url_ssl": "https://example.com/assets/notifier/profile-avatar.png", + "recruiter_id": None, + "rnc_id": None, + "rnc_regid": None, + "salesforce_id": None, + "school_district": None, + "school_sub_district": None, + "sex": None, + "signup_type": 0, + "state_file_id": None, + "state_lower_district": None, + "state_upper_district": None, + "support_level": None, + "supranational_district": None, + "tags": ["zoo", "bar"], + "twitter_id": None, + "twitter_name": None, + "updated_at": "2023-06-22T11:43:03-07:00", + "van_id": None, + "village_district": None, + "ward": None, + "work_phone_number": None, + }, + ], + "next": None, + "prev": None, +} + +PERSON_RESPONSE = { + "person": { + "birthdate": None, + "city_district": None, + "civicrm_id": None, + "county_district": None, + "county_file_id": None, + "created_at": "2023-06-22T08:21:00-07:00", + "datatrust_id": None, + "do_not_call": False, + "do_not_contact": False, + "dw_id": None, + "email": "foo@example.com", + "email_opt_in": True, + "employer": None, + "external_id": None, + "federal_district": None, + "fire_district": None, + "first_name": "Foo", + "has_facebook": False, + "id": 1, + "is_twitter_follower": False, + "is_volunteer": False, + "judicial_district": None, + "labour_region": None, + "last_name": "Bar", + "linkedin_id": None, + "mobile": None, + "mobile_opt_in": True, + "middle_name": "", + "nbec_guid": None, + "ngp_id": None, + "note": None, + "occupation": None, + "party": None, + "pf_strat_id": None, + "phone": None, + "precinct_id": None, + "primary_address": None, + "profile_image_url_ssl": "https://example.com/assets/notifier/profile-avatar.png", + "recruiter_id": None, + "rnc_id": None, + "rnc_regid": None, + "salesforce_id": None, + "school_district": None, + "school_sub_district": None, + "sex": None, + "signup_type": 0, + "state_file_id": None, + "state_lower_district": None, + "state_upper_district": None, + "support_level": None, + "supranational_district": None, + "tags": [], + "twitter_id": None, + "twitter_name": None, + "updated_at": "2023-06-22T08:21:00-07:00", + "van_id": None, + "village_district": None, + "ward": None, + "work_phone_number": None, + "active_customer_expires_at": None, + "active_customer_started_at": None, + "author": None, + "author_id": None, + "auto_import_id": None, + "availability": None, + "ballots": [], + "banned_at": None, + "billing_address": None, + "bio": None, + "call_status_id": None, + "call_status_name": None, + "capital_amount_in_cents": 500, + "children_count": 0, + "church": None, + "city_sub_district": None, + "closed_invoices_amount_in_cents": None, + "closed_invoices_count": None, + "contact_status_id": None, + "contact_status_name": None, + "could_vote_status": False, + "demo": None, + "donations_amount_in_cents": 0, + "donations_amount_this_cycle_in_cents": 0, + "donations_count": 0, + "donations_count_this_cycle": 0, + "donations_pledged_amount_in_cents": 0, + "donations_raised_amount_in_cents": 0, + "donations_raised_amount_this_cycle_in_cents": 0, + "donations_raised_count": 0, + "donations_raised_count_this_cycle": 0, + "donations_to_raise_amount_in_cents": 0, + "email1": "foo@example.com", + "email1_is_bad": False, + "email2": None, + "email2_is_bad": False, + "email3": None, + "email3_is_bad": False, + "email4": None, + "email4_is_bad": False, + "emails": [ + { + "email_address": "foo@example.com", + "email_number": 1, + "is_bad": False, + "is_primary": True, + } + ], + "ethnicity": None, + "facebook_address": None, + "facebook_profile_url": None, + "facebook_updated_at": None, + "facebook_username": None, + "fax_number": None, + "federal_donotcall": False, + "first_donated_at": None, + "first_fundraised_at": None, + "first_invoice_at": None, + "first_prospect_at": None, + "first_recruited_at": None, + "first_supporter_at": "2023-06-22T08:21:00-07:00", + "first_volunteer_at": None, + "full_name": "Foo Bar", + "home_address": None, + "import_id": None, + "inferred_party": None, + "inferred_support_level": None, + "invoice_payments_amount_in_cents": None, + "invoice_payments_referred_amount_in_cents": None, + "invoices_amount_in_cents": None, + "invoices_count": None, + "is_absentee_voter": None, + "is_active_voter": None, + "is_deceased": False, + "is_donor": False, + "is_dropped_from_file": None, + "is_early_voter": None, + "is_fundraiser": False, + "is_ignore_donation_limits": False, + "is_leaderboardable": True, + "is_mobile_bad": False, + "is_permanent_absentee_voter": None, + "is_possible_duplicate": False, + "is_profile_private": False, + "is_profile_searchable": True, + "is_prospect": False, + "is_supporter": True, + "is_survey_question_private": False, + "language": None, + "last_call_id": None, + "last_contacted_at": None, + "last_contacted_by": None, + "last_donated_at": None, + "last_fundraised_at": None, + "last_invoice_at": None, + "last_rule_violation_at": None, + "legal_name": None, + "locale": None, + "mailing_address": None, + "marital_status": None, + "media_market_name": None, + "meetup_id": None, + "meetup_address": None, + "mobile_normalized": None, + "nbec_precinct_code": None, + "nbec_precinct": None, + "note_updated_at": None, + "outstanding_invoices_amount_in_cents": None, + "outstanding_invoices_count": None, + "overdue_invoices_count": None, + "page_slug": None, + "parent": None, + "parent_id": None, + "party_member": None, + "phone_normalized": None, + "phone_time": None, + "precinct_code": None, + "precinct_name": None, + "prefix": None, + "previous_party": None, + "primary_email_id": 1, + "priority_level": None, + "priority_level_changed_at": None, + "profile_content": None, + "profile_content_html": None, + "profile_headline": None, + "received_capital_amount_in_cents": 500, + "recruiter": None, + "recruits_count": 0, + "registered_address": None, + "registered_at": None, + "religion": None, + "rule_violations_count": 0, + "signup_sources": [], + "spent_capital_amount_in_cents": 0, + "submitted_address": None, + "subnations": [], + "suffix": None, + "support_level_changed_at": None, + "support_probability_score": None, + "township": None, + "turnout_probability_score": None, + "twitter_address": None, + "twitter_description": None, + "twitter_followers_count": None, + "twitter_friends_count": None, + "twitter_location": None, + "twitter_login": None, + "twitter_updated_at": None, + "twitter_website": None, + "unsubscribed_at": None, + "user_submitted_address": None, + "username": None, + "voter_updated_at": None, + "warnings_count": 0, + "website": None, + "work_address": None, + }, + "precinct": None, +} diff --git a/test/test_nation_builder/test_nation_builder.py b/test/test_nation_builder/test_nation_builder.py new file mode 100644 index 0000000000..83c52ffe6f --- /dev/null +++ b/test/test_nation_builder/test_nation_builder.py @@ -0,0 +1,178 @@ +import unittest + +import requests_mock + +from parsons import NationBuilder as NB + +from .fixtures import GET_PEOPLE_RESPONSE, PERSON_RESPONSE + + +class TestNationBuilder(unittest.TestCase): + def test_client(self): + nb = NB("test-slug", "test-token") + self.assertEqual(nb.client.uri, "https://test-slug.nationbuilder.com/api/v1/") + self.assertEqual( + nb.client.headers, + { + "authorization": "Bearer test-token", + "Content-Type": "application/json", + "Accept": "application/json", + }, + ) + + def test_get_uri_success(self): + self.assertEqual(NB.get_uri("foo"), "https://foo.nationbuilder.com/api/v1") + self.assertEqual(NB.get_uri("bar"), "https://bar.nationbuilder.com/api/v1") + + def test_get_uri_errors(self): + values = ["", " ", None, 1337, {}, []] + + for v in values: + with self.assertRaises(ValueError): + NB.get_uri(v) + + def test_get_auth_headers_success(self): + self.assertEqual(NB.get_auth_headers("foo"), {"authorization": "Bearer foo"}) + self.assertEqual(NB.get_auth_headers("bar"), {"authorization": "Bearer bar"}) + + def test_get_auth_headers_errors(self): + values = ["", " ", None, 1337, {}, []] + + for v in values: + with self.assertRaises(ValueError): + NB.get_auth_headers(v) + + def test_parse_next_params_success(self): + n, t = NB.parse_next_params("/a/b/c?__nonce=foo&__token=bar") + self.assertEqual(n, "foo") + self.assertEqual(t, "bar") + + def test_get_next_params_errors(self): + with self.assertRaises(ValueError): + NB.parse_next_params("/a/b/c?baz=1") + + with self.assertRaises(ValueError): + NB.parse_next_params("/a/b/c?__nonce=1") + + with self.assertRaises(ValueError): + NB.parse_next_params("/a/b/c?__token=1") + + def test_make_next_url(self): + self.assertEqual( + NB.make_next_url("example.com", "bar", "baz"), + "example.com?limit=100&__nonce=bar&__token=baz", + ) + + @requests_mock.Mocker() + def test_get_people_handle_empty_response(self, m): + nb = NB("test-slug", "test-token") + m.get("https://test-slug.nationbuilder.com/api/v1/people", json={"results": []}) + table = nb.get_people() + self.assertEqual(table.num_rows, 0) + + @requests_mock.Mocker() + def test_get_people(self, m): + nb = NB("test-slug", "test-token") + m.get( + "https://test-slug.nationbuilder.com/api/v1/people", + json=GET_PEOPLE_RESPONSE, + ) + table = nb.get_people() + + self.assertEqual(table.num_rows, 2) + self.assertEqual(len(table.columns), 59) + + self.assertEqual(table[0]["first_name"], "Foo") + self.assertEqual(table[0]["last_name"], "Bar") + self.assertEqual(table[0]["email"], "foo@example.com") + + @requests_mock.Mocker() + def test_get_people_with_next(self, m): + """Make two requests and get the same data twice. This will exercise the while loop.""" + nb = NB("test-slug", "test-token") + + GET_PEOPLE_RESPONSE_WITH_NEXT = GET_PEOPLE_RESPONSE.copy() + GET_PEOPLE_RESPONSE_WITH_NEXT[ + "next" + ] = "https://test-slug.nationbuilder.com/api/v1/people?limit=100&__nonce=bar&__token=baz" + + m.get( + "https://test-slug.nationbuilder.com/api/v1/people", + json=GET_PEOPLE_RESPONSE_WITH_NEXT, + ) + + m.get( + "https://test-slug.nationbuilder.com/api/v1/people?limit=100&__nonce=bar&__token=baz", + json=GET_PEOPLE_RESPONSE, + ) + + table = nb.get_people() + + self.assertEqual(table.num_rows, 4) + self.assertEqual(len(table.columns), 59) + + self.assertEqual(table[1]["first_name"], "Zoo") + self.assertEqual(table[1]["last_name"], "Baz") + self.assertEqual(table[1]["email"], "bar@example.com") + + def test_update_person_raises_with_bad_params(self): + nb = NB("test-slug", "test-token") + + with self.assertRaises(ValueError): + nb.update_person(None, {}) + + with self.assertRaises(ValueError): + nb.update_person(1, {}) + + with self.assertRaises(ValueError): + nb.update_person(" ", {}) + + with self.assertRaises(ValueError): + nb.update_person("1", None) + + with self.assertRaises(ValueError): + nb.update_person("1", "bad value") + + @requests_mock.Mocker() + def test_update_person(self, m): + """Requests the correct URL, returns the correct data and doesn't raise exceptions.""" + nb = NB("test-slug", "test-token") + + m.put( + "https://test-slug.nationbuilder.com/api/v1/people/1", + json=PERSON_RESPONSE, + ) + + response = nb.update_person("1", {"tags": ["zoot", "boot"]}) + person = response["person"] + + self.assertEqual(person["id"], 1) + self.assertEqual(person["first_name"], "Foo") + self.assertEqual(person["last_name"], "Bar") + self.assertEqual(person["email"], "foo@example.com") + + def test_upsert_person_raises_with_bad_params(self): + nb = NB("test-slug", "test-token") + + with self.assertRaises(ValueError): + nb.upsert_person({"tags": ["zoot", "boot"]}) + + @requests_mock.Mocker() + def test_upsert_person(self, m): + """Requests the correct URL, returns the correct data and doesn't raise exceptions.""" + nb = NB("test-slug", "test-token") + + m.put( + "https://test-slug.nationbuilder.com/api/v1/people/push", + json=PERSON_RESPONSE, + ) + + created, response = nb.upsert_person({"email": "foo@example.com"}) + self.assertFalse(created) + + person = response["person"] + + self.assertEqual(person["id"], 1) + self.assertEqual(person["first_name"], "Foo") + self.assertEqual(person["last_name"], "Bar") + self.assertEqual(person["email"], "foo@example.com")