Fix assert_matching_tables (#759)

* Fix tests * Only use .items() to assert matching tables for dicts * Fix test_aws_async tests The csv_renewed and distribute_task functions return a table with the ints converted to strs * Fix match_columns The function definition of match_column states that if there are 2 columns with the same normalized name, the latter should be removed by default. The normalize_fn function must run over the reversed list of self.columns in order for the first matching column to be kept. In it's previous state, the last matching column was kept * Fix order of records To match order in airtable_responses.py * Fix Hustle create_leads test Current test is returning the Lyndon Johnson lead twice * Fix test_targets.py Fix typo in test. Also change import unittest to import unittest.mock to avoid `AttributeError: module 'unittest' has no attribute 'mock'` when running test locally * Figure out if r1 and r2 are lists separately
move-coop · Oct 20, 2022 · 17df1da · 17df1da
1 parent 36efb19
commit 17df1da
Show file tree

Hide file tree

Showing 8 changed files with 54 additions and 16 deletions.
diff --git a/parsons/etl/etl.py b/parsons/etl/etl.py
@@ -852,7 +852,7 @@ def match_columns(self, desired_columns, fuzzy_match=True, if_extra_columns='rem
 
         # Create a mapping of our "normalized" name to the original column name
         current_columns_normalized = {
-            normalize_fn(col): col for col in self.columns
+            normalize_fn(col): col for col in reversed(self.columns)
         }
 
         # Track any columns we need to add to our current table from our desired columns

diff --git a/test/test_airtable/test_airtable.py b/test/test_airtable/test_airtable.py
@@ -41,12 +41,12 @@ def test_get_records(self, m):
 
         m.get(self.base_uri, json=records_response)
 
-        tbl = Table([{'id': 'recObtmLUrD5dOnmD',
-                      'createdTime': '2019-05-08T19:37:58.000Z',
-                      'Name': None},
-                     {'id': 'recaBMSHTgXREa5ef',
+        tbl = Table([{'id': 'recaBMSHTgXREa5ef',
                       'createdTime': '2019-05-08T19:37:58.000Z',
                       'Name': 'This is a row!'},
+                     {'id': 'recObtmLUrD5dOnmD',
+                      'createdTime': '2019-05-08T19:37:58.000Z',
+                      'Name': None},
                      {'id': 'recmeBNnj4cuHPOSI',
                       'createdTime': '2019-05-08T19:37:58.000Z',
                       'Name': None}])

diff --git a/test/test_aws_async.py b/test/test_aws_async.py
@@ -49,7 +49,7 @@ def test_task_path_conversion(self):
         print('csv_str', csv_str)
         csv_renewed = import_and_get_task(csv_str)
         assert_matching_tables(csv_renewed('x,y\n1,2'),
-                               Table([('x', 'y'), (1, 2)]))
+                               Table([('x', 'y'), ('1', '2')]))
 
         # Table.to_dicts (instance)
         dicts_str = get_func_task_path(Table.to_dicts, Table)
@@ -79,7 +79,7 @@ def test_distribute_task(self):
         assert_matching_tables(
             tableargs[0],
             Table([('x', 'y'),
-                   (11, 12), (13, 14), (15, 16), (17, 18), (19, 10)]))
+                   ('11', '12'), ('13', '14'), ('15', '16'), ('17', '18'), ('19', '10')]))
         count = 0
         tableargs = None
         distribute_task(Table(datatable + [(0, 0)]),
@@ -95,7 +95,7 @@ def test_distribute_task(self):
         self.assertEqual(tableargs[1:], ('initx', 1, 2, 3))
         assert_matching_tables(
             tableargs[0],
-            Table([('x', 'y'), (0, 0)]))
+            Table([('x', 'y'), ('0', '0')]))
 
         # 3. catch=True (with throwing)
         count = 0

diff --git a/test/test_google/test_google_admin.py b/test/test_google/test_google_admin.py
@@ -18,7 +18,7 @@ class TestGoogleAdmin(unittest.TestCase):
     mock_all_groups = Table([
         {'email': 'fakeemail4@fakedomain.com', 'id': 1},
         {'email': 'fakeemail5@fakedomain.com', 'id': 2},
-        {'email': 'fakeemail6@fakedomain.com', 'id': 2}
+        {'email': 'fakeemail6@fakedomain.com', 'id': 3}
     ])
 
     def setUp(self):

diff --git a/test/test_hustle/expected_json.py b/test/test_hustle/expected_json.py
@@ -151,6 +151,36 @@
     'createdAt': '2019-09-20T22:15:48.033Z'
 }
 
+created_leads = [{
+    'id': 'yK5jo2tlms',
+    'type': 'Lead',
+    'customFields': {
+        'address': '123 Main Street'
+    },
+    'globalOptedOut': False,
+    'groupIds': ['cMCH0hxwGt'],
+    'firstName': 'Lyndon',
+    'lastName': 'Johnson',
+    'organizationId': 'LePEoKzD3Z',
+    'phoneNumber': '+14435705355',
+    'tagIds': [],
+    'createdAt': '2019-09-20T22:15:46.706Z'
+}, {
+    'id': 't18JdlHW7r',
+    'type': 'Lead',
+    'customFields': {
+        'address': '124 Main Street'
+    },
+    'globalOptedOut': False,
+    'groupIds': ['cMCH0hxwGt'],
+    'firstName': 'Ann',
+    'lastName': 'Richards',
+    'organizationId': 'LePEoKzD3Z',
+    'phoneNumber': '+14435705354',
+    'tagIds': [],
+    'createdAt': '2019-09-20T22:15:48.033Z'
+}]
+
 leads = {
     'items': [{
         'id': 'wqy78hlz2',

diff --git a/test/test_hustle/test_hustle.py b/test/test_hustle/test_hustle.py
@@ -60,13 +60,15 @@ def test_create_lead(self, m):
     @requests_mock.Mocker()
     def test_create_leads(self, m):
 
-        m.post(HUSTLE_URI + 'groups/cMCH0hxwGt/leads', json=expected_json.leads_tbl_01)
+        m.post(HUSTLE_URI + 'groups/cMCH0hxwGt/leads', [
+            {'json': expected_json.leads_tbl_01}, {'json': expected_json.leads_tbl_02}
+        ])
 
         tbl = Table([['phone_number', 'ln', 'first_name'],
-                     ['4435705355', 'Warren', 'Elizabeth'],
-                     ['5126993336', 'Obama', 'Barack']])
+                     ['4435705355', 'Johnson', 'Lyndon'],
+                     ['4435705354', 'Richard', 'Ann']])
         ids = self.hustle.create_leads(tbl, group_id='cMCH0hxwGt')
-        assert_matching_tables(ids, Table(expected_json.leads['items']))
+        assert_matching_tables(ids, Table(expected_json.created_leads))
 
     @requests_mock.Mocker()
     def test_update_lead(self, m):

diff --git a/test/test_van/test_targets.py b/test/test_van/test_targets.py
@@ -1,4 +1,4 @@
-import unittest
+import unittest.mock
 import os
 import requests_mock
 from parsons import VAN, Table
@@ -116,7 +116,7 @@ def test_get_target_export(self, fromcsv, m):
 
         expected_result = Table([
             ('TargetID', 'TargetName', 'TargetSubgroupID', 'TargetSubgroupName', 'VanID'),
-            ('12827', 'Volunteer Recruitment Tiers', '1111', 'Tier', '109957740'),
+            ('12827', 'Volunteer Recruitment Tiers', '1111', 'Tier', '109957749'),
             ('12827', 'Volunteer Recruitment Tiers', '1111', 'Tier', '109957754')])
 
         assert_matching_tables(self.van.get_target_export(export_job_id), expected_result)

diff --git a/test/utils.py b/test/utils.py
@@ -37,5 +37,11 @@ def assert_matching_tables(table1, table2, ignore_headers=False):
         assert data1.num_rows == data2.num_rows
 
     for r1, r2 in zip(data1, data2):
-        # Cast both rows to lists, in case they are different types of collections
+        # Cast both rows to lists, in case they are different types of collections. Must call
+        # .items() on dicts to compare content of collections
+        if isinstance(r1, dict):
+            r1 = r1.items()
+        if isinstance(r2, dict):
+            r2 = r2.items()
+
         assert list(r1) == list(r2)