From 77e352823e945f7931d5e3d8f15b8dc7249792ad Mon Sep 17 00:00:00 2001 From: Cody Gordon Date: Fri, 14 Jun 2024 09:16:51 -0400 Subject: [PATCH 1/2] van bulk import - add apply canvass results & custom contact fields / docstring fixes / clarification / contact mapping --- parsons/ngpvan/bulk_import.py | 153 +++++++++++++++++++++++++++++-- test/test_van/test_bulkimport.py | 39 ++++++++ 2 files changed, 185 insertions(+), 7 deletions(-) diff --git a/parsons/ngpvan/bulk_import.py b/parsons/ngpvan/bulk_import.py index c0c7ddb0ad..714652b726 100644 --- a/parsons/ngpvan/bulk_import.py +++ b/parsons/ngpvan/bulk_import.py @@ -189,6 +189,9 @@ def bulk_apply_activist_codes(self, tbl, url_type, **url_kwargs): * - ``datecanvassed`` - No - An ISO formatted date + * - ``canvassedby`` + - No + - A valid User ID; Required when DateCanvassed is provided * - ``contacttypeid`` - No - The method of contact. @@ -249,7 +252,7 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs): - - * - First Name - - ``fn``, ``firstname``, ``last`` + - ``fn``, ``firstname``, ``first`` - * - Middle Name - ``mn``, ``middlename``, ``middle`` @@ -258,8 +261,8 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs): - ``ln``, ``lastname``, ``last`` - * - Date of Birth - - ``dob``, ``dateofbirth`` ``birthdate`` - - What type of thing does this need? + - ``dob``, ``dateofbirth``, ``birthdate`` + - An ISO formatted date * - Sex - ``sex``, ``gender`` - @@ -281,6 +284,9 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs): * - State Or Province - ``state``, ``st``, ``stateorprovince`` - + * - Zip or Postal Code + - ``ziporpostal``, ``postal``, ``postalcode``, ``zip``, ``zipcode`` + - * - Country Code - ``countrycode``, ``country`` - A valid two character country code (e.g. ``US``) @@ -309,6 +315,9 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs): * - Email - ``email``, ``emailaddress`` - + * - Other Email + - ``otheremail``, ``email2``, ``emailaddress2`` + - `Args:` table: Parsons table @@ -325,7 +334,7 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs): The bulk import job id """ - tbl = tbl.map_columns(COLUMN_MAP, exact_match=False) + tbl = tbl.map_columns(CONTACTS_COLUMN_MAP, exact_match=False) return self.post_bulk_import( tbl, @@ -339,13 +348,13 @@ def bulk_upsert_contacts(self, tbl, url_type, result_fields=None, **url_kwargs): def bulk_apply_suppressions(self, tbl, url_type, **url_kwargs): """ - Bulk apply contact suppressions codes. + Bulk apply contact suppression codes. The table may include the following columns. The first column must be ``vanid``. .. list-table:: - :widths: 25 25 + :widths: 25 25 50 :header-rows: 1 * - Column Name @@ -381,11 +390,139 @@ def bulk_apply_suppressions(self, tbl, url_type, **url_kwargs): **url_kwargs, ) + def bulk_apply_canvass_results(self, tbl, url_type, **url_kwargs): + """ + Bulk apply contact canvass results. + + The table may include the following columns. The first column + must be ``vanid``. + + .. list-table:: + :widths: 25 25 50 + :header-rows: 1 + + * - Column Name + - Required + - Description + * - ``vanid`` + - Yes + - A valid VANID primary key + * - ``contacttypeid`` + - Yes + - Valid Contact Type ID + * - ``resultid`` + - Yes + - Valid Contact Result ID + * - ``datecanvassed`` + - Yes + - ISO Date Format + * - ``canvassedby`` + - Yes + - Valid User ID + * - ``phone`` + - No + - Attempted Phone Number + * - ``countrycode`` + - No + - Country Code (ISO 3166-1 alpha-2) + * - ``phonetypeid`` + - No + - Phone Type + * - ``phoneoptinstatusid`` + - No + - SMS Opt-In Status + * - ``addressid`` + - No + - The Contact Address ID of the address that was canvassed + + `Args:` + table: Parsons table + A Parsons table. + url_type: str + The cloud file storage to use to post the file (``S3`` or ``GCS``). + See :ref:`Cloud Storage ` for more details. + **url_kwargs: kwargs + Arguments to configure your cloud storage url type. See + :ref:`Cloud Storage ` for more details. + `Returns:` + int + The bulk import job id + """ + + return self.post_bulk_import( + tbl, + url_type, + "Contacts", + [{"name": "CanvassResults"}], + "Apply Canvass Results", + **url_kwargs, + ) + + def bulk_apply_contact_custom_fields(self, custom_field_group_id, tbl, url_type, **url_kwargs): + """ + Bulk apply contact custom fields. + + The table may include the following columns. The first column + must be ``vanid``. + + .. list-table:: + :widths: 25 25 60 + :header-rows: 1 + + * - Column Name + - Required + - Description + * - ``vanid`` + - Yes + - A valid VANID primary key + * - ***``CF{CustomFieldID}`` + - Yes + - At least one custom field column to be loaded associated with the provided custom_field_group_id. + The column name should be a valid Custom Field ID prefixed with ``CF``, i.e. CF123. + + `Args:` + custom_field_group_id: int + Valid Custom Contact Field Group ID; must be the parent of + the provided Custom Field IDs in the file. + table: Parsons table + A Parsons table. + url_type: str + The cloud file storage to use to post the file (``S3`` or ``GCS``). + See :ref:`Cloud Storage ` for more details. + **url_kwargs: kwargs + Arguments to configure your cloud storage url type. See + :ref:`Cloud Storage ` for more details. + `Returns:` + int + The bulk import job id + """ + + mapping_types = [ + { + "name": "ApplyContactCustomFields", + "fieldValueMappings": [ + { + "fieldName": "CustomFieldGroupID", + "staticValue": custom_field_group_id, + }, + ], + } + ] + + return self.post_bulk_import( + tbl, + url_type, + "Contacts", + mapping_types, + "Apply Contact Custom Fields", + **url_kwargs, + ) + # This is a column mapper that is used to accept additional column names and provide # flexibility for the user. -COLUMN_MAP = { +CONTACTS_COLUMN_MAP = { "firstname": ["fn", "first"], "middlename": ["mn", "middle"], "lastname": ["ln", "last"], @@ -396,6 +533,7 @@ def bulk_apply_suppressions(self, tbl, url_type, **url_kwargs): "addressline3": ["addressline3", "address3"], "city": [], "stateorprovince": ["state", "st"], + "ziporpostal": ["postal", "postalcode", "zip", "zipcode"], "countrycode": ["country"], "displayasentered": [], "cellphone": ["cell"], @@ -403,4 +541,5 @@ def bulk_apply_suppressions(self, tbl, url_type, **url_kwargs): "phone": ["home", "homephone"], "phonecountrycode": ["phonecountrycode"], "email": ["emailaddress"], + "otheremail": ["email2", "emailaddress2"], } diff --git a/test/test_van/test_bulkimport.py b/test/test_van/test_bulkimport.py index f667d2e990..a090ec2f17 100644 --- a/test/test_van/test_bulkimport.py +++ b/test/test_van/test_bulkimport.py @@ -164,6 +164,45 @@ def test_bulk_upsert_contacts(self, m): self.assertEqual(job_id, 54679) + @requests_mock.Mocker() + def test_bulk_apply_canvass_results(self, m): + + # Mock Cloud Storage + cloud_storage.post_file = mock.MagicMock() + cloud_storage.post_file.return_value = "https://s3.com/my_file.zip" + + tbl = Table( + [ + ["vanid", "contacttypeid", "resultid", "datecanvassed", "canvassedby", "phone"], + [1234, 1, 1, "2020-01-01", 987, "5554443210"], + ] + ) + + m.post(self.van.connection.uri + "bulkImportJobs", json={"jobId": 54679}) + + job_id = self.van.bulk_apply_canvass_results(tbl, url_type="S3", bucket="my-bucket") + + self.assertEqual(job_id, 54679) + + @requests_mock.Mocker() + def test_bulk_apply_contact_custom_fields(self, m): + + # Mock Cloud Storage + cloud_storage.post_file = mock.MagicMock() + cloud_storage.post_file.return_value = "https://s3.com/my_file.zip" + + tbl = Table([["vanid", "CF123", "CF124"], [1234, "Test String Value", 999]]) + + m.post(self.van.connection.uri + "bulkImportJobs", json={"jobId": 54679}) + + custom_field_group_id = 1234 + + job_id = self.van.bulk_apply_contact_custom_fields( + custom_field_group_id, tbl, url_type="S3", bucket="my-bucket" + ) + + self.assertEqual(job_id, 54679) + mapping_type = { "name": "ActivistCode", From 4a8f8934758728b76e411679862ad4f072fd12c7 Mon Sep 17 00:00:00 2001 From: Cody Gordon Date: Fri, 14 Jun 2024 12:18:29 -0400 Subject: [PATCH 2/2] van bulk import additions - fix E501 --- parsons/ngpvan/bulk_import.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/parsons/ngpvan/bulk_import.py b/parsons/ngpvan/bulk_import.py index 714652b726..10e73a01c9 100644 --- a/parsons/ngpvan/bulk_import.py +++ b/parsons/ngpvan/bulk_import.py @@ -477,8 +477,9 @@ def bulk_apply_contact_custom_fields(self, custom_field_group_id, tbl, url_type, - A valid VANID primary key * - ***``CF{CustomFieldID}`` - Yes - - At least one custom field column to be loaded associated with the provided custom_field_group_id. - The column name should be a valid Custom Field ID prefixed with ``CF``, i.e. CF123. + - At least one custom field column to be loaded associated with the provided + custom_field_group_id. The column name should be a valid Custom Field ID + prefixed with ``CF``, i.e. CF123. `Args:` custom_field_group_id: int