chore: Add py3.10 to compatibility matrix across all services except …

…for metadata (#2242) * Add py3.10 to compatibility matrix Signed-off-by: Kristen Armes <karmes@lyft.com> * Fix search tests and upgrade pyarrow Signed-off-by: Kristen Armes <karmes@lyft.com> * One more search test fix Signed-off-by: Kristen Armes <karmes@lyft.com> * lint Signed-off-by: Kristen Armes <karmes@lyft.com> * Take 3.10 out for metadata, currently blocked by amundsengremlin related tests Signed-off-by: Kristen Armes <karmes@lyft.com> --------- Signed-off-by: Kristen Armes <karmes@lyft.com>
amundsen-io · Apr 4, 2024 · 164b956 · 164b956
1 parent 7a8fcba
commit 164b956
Show file tree

Hide file tree

Showing 10 changed files with 45 additions and 52 deletions.
diff --git a/.github/workflows/common_pull_request.yml b/.github/workflows/common_pull_request.yml
@@ -9,7 +9,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.8.x', '3.9.x']
+        python-version: ['3.8.x', '3.9.x', '3.10.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1

diff --git a/.github/workflows/databuilder_pull_request.yml b/.github/workflows/databuilder_pull_request.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.8.x', '3.9.x']
+        python-version: ['3.8.x', '3.9.x', '3.10.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1

diff --git a/.github/workflows/frontend_pull_request.yml b/.github/workflows/frontend_pull_request.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.8.x', '3.9.x']
+        python-version: ['3.8.x', '3.9.x', '3.10.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1

diff --git a/.github/workflows/noop_pull_request.yml b/.github/workflows/noop_pull_request.yml
@@ -30,7 +30,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8.x', '3.9.x']
+        python-version: ['3.8.x', '3.9.x', '3.10.x']
     steps:
       - name: Do nothing
         run: exit 0
diff --git a/.github/workflows/search_pull_request.yml b/.github/workflows/search_pull_request.yml
@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-22.04
     strategy:
       matrix:
-        python-version: ['3.8.x', '3.9.x']
+        python-version: ['3.8.x', '3.9.x', '3.10.x']
     steps:
       - name: Checkout
         uses: actions/checkout@v1

diff --git a/common/setup.py b/common/setup.py
@@ -51,5 +51,6 @@
     classifiers=[
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ],
 )
diff --git a/databuilder/setup.py b/databuilder/setup.py
@@ -142,5 +142,6 @@
     classifiers=[
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ],
 )
diff --git a/frontend/setup.py b/frontend/setup.py
@@ -50,7 +50,7 @@ def build_js() -> None:
 jira = ['jira==3.0.1']
 asana = ['asana==0.10.3']
 oidc = ['flaskoidc>=1.0.0']
-pyarrow = ['pyarrow==3.0.0']
+pyarrow = ['pyarrow==15.0.0']
 bigquery_preview = ['google-cloud-bigquery>=2.13.1,<3.0.0', 'flatten-dict==0.3.0']
 all_deps = requirements + requirements_common + requirements_dev + oidc + pyarrow + bigquery_preview + jira + asana
 
@@ -83,5 +83,6 @@ def build_js() -> None:
     classifiers=[
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ],
 )
diff --git a/search/setup.py b/search/setup.py
@@ -44,5 +44,6 @@
     classifiers=[
         'Programming Language :: Python :: 3.8',
         'Programming Language :: Python :: 3.9',
+        'Programming Language :: Python :: 3.10',
     ],
 )
diff --git a/search/tests/unit/proxy/test_elasticsearch.py b/search/tests/unit/proxy/test_elasticsearch.py
@@ -5,7 +5,6 @@
 from typing import (  # noqa: F401
     Any, Iterable, List, Optional,
 )
-from unittest import mock
 from unittest.mock import MagicMock, patch
 
 from amundsen_common.models.api import health_check
@@ -548,7 +547,6 @@ def test_create_document(self, mock_uuid: MagicMock) -> None:
             {
                 'index': {
                     '_index': new_index_name,
-                    '_type': 'table',
                     '_id': 'snowflake://blue.test_schema/bank_accounts'
                 }
             },
@@ -569,11 +567,11 @@ def test_create_document(self, mock_uuid: MagicMock) -> None:
                 'total_usage': 0,
                 'programmatic_descriptions': None,
                 'schema_description': 'schema description 1',
+                'resource_type': 'table',
             },
             {
                 'index': {
                     '_index': new_index_name,
-                    '_type': 'table',
                     '_id': 'snowflake://blue.test_schema/bitcoin_wallets'
                 }
             },
@@ -593,20 +591,17 @@ def test_create_document(self, mock_uuid: MagicMock) -> None:
                 'badges': [],
                 'total_usage': 0,
                 'schema_description': 'schema description 2',
-                'programmatic_descriptions': ["test"]
+                'programmatic_descriptions': ["test"],
+                'resource_type': 'table',
             }
         ]
 
-        _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias)
-        _get_alias.return_value = dict([(new_index_name, {})])
-
-        _bulk = mock.create_autospec(mock_elasticsearch.indices.get_alias)
-        _bulk.return_value = {'errors': False}
-
         expected_alias = 'table_search_index'
-        result = self.es_proxy.create_document(data=start_data, index=expected_alias)
-        self.assertEqual(expected_alias, result)
-        _bulk.assert_called_with(body=expected_data)
+        with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \
+                patch.object(mock_elasticsearch, 'bulk', return_value={'errors': False}) as bulk:
+            result = self.es_proxy.create_document(data=start_data, index=expected_alias)
+            self.assertEqual(expected_alias, result)
+            bulk.assert_called_with(body=expected_data)
 
     def test_update_document_with_no_data(self) -> None:
         expected = ''
@@ -617,8 +612,6 @@ def test_update_document_with_no_data(self) -> None:
     def test_update_document(self, mock_uuid: MagicMock) -> None:
         mock_elasticsearch = self.es_proxy.elasticsearch
         new_index_name = 'tester_index_name'
-        _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias)
-        _get_alias.return_value = dict([(new_index_name, {})])
         mock_uuid.return_value = new_index_name
         table_key = 'snowflake://blue.test_schema/bitcoin_wallets'
         expected_alias = 'table_search_index'
@@ -633,7 +626,6 @@ def test_update_document(self, mock_uuid: MagicMock) -> None:
             {
                 'update': {
                     '_index': new_index_name,
-                    '_type': 'table',
                     '_id': table_key
                 }
             },
@@ -655,75 +647,72 @@ def test_update_document(self, mock_uuid: MagicMock) -> None:
                     'total_usage': 0,
                     'programmatic_descriptions': None,
                     'schema_description': 'schema description 1',
+                    'resource_type': 'table',
                 }
             }
         ]
-        result = self.es_proxy.update_document(data=data, index=expected_alias)
-        self.assertEqual(expected_alias, result)
-        _bulk = mock.create_autospec(mock_elasticsearch.bulk)
-        _bulk.assert_called_with(body=expected_data)
+        with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \
+                patch.object(mock_elasticsearch, 'bulk') as bulk:
+            result = self.es_proxy.update_document(data=data, index=expected_alias)
+            self.assertEqual(expected_alias, result)
+            bulk.assert_called_with(body=expected_data)
 
     @patch('uuid.uuid4')
     def test_delete_table_document(self, mock_uuid: MagicMock) -> None:
         mock_elasticsearch = self.es_proxy.elasticsearch
         new_index_name = 'tester_index_name'
         mock_uuid.return_value = new_index_name
-
-        _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias)
-        _get_alias.return_value = dict([(new_index_name, {})])
-
         expected_alias = 'table_search_index'
         data = ['id1', 'id2']
 
         expected_data = [
-            {'delete': {'_index': new_index_name, '_id': 'id1', '_type': 'table'}},
-            {'delete': {'_index': new_index_name, '_id': 'id2', '_type': 'table'}}
+            {'delete': {'_index': new_index_name, '_id': 'id1'}},
+            {'delete': {'_index': new_index_name, '_id': 'id2'}}
         ]
-        result = self.es_proxy.delete_document(data=data, index=expected_alias)
 
-        self.assertEqual(expected_alias, result)
-        _bulk = mock.create_autospec(mock_elasticsearch.bulk)
-        _bulk.assert_called_with(body=expected_data)
+        with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \
+                patch.object(mock_elasticsearch, 'bulk') as bulk:
+            result = self.es_proxy.delete_document(data=data, index=expected_alias)
+            self.assertEqual(expected_alias, result)
+            bulk.assert_called_with(body=expected_data)
 
     @patch('uuid.uuid4')
     def test_delete_user_document(self, mock_uuid: MagicMock) -> None:
         mock_elasticsearch = self.es_proxy.elasticsearch
         new_index_name = 'tester_index_name'
         mock_uuid.return_value = new_index_name
-        _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias)
-        _get_alias.return_value = dict([(new_index_name, {})])
         expected_alias = 'user_search_index'
         data = ['id1', 'id2']
 
         expected_data = [
-            {'delete': {'_index': new_index_name, '_id': 'id1', '_type': 'user'}},
-            {'delete': {'_index': new_index_name, '_id': 'id2', '_type': 'user'}}
+            {'delete': {'_index': new_index_name, '_id': 'id1'}},
+            {'delete': {'_index': new_index_name, '_id': 'id2'}}
         ]
-        result = self.es_proxy.delete_document(data=data, index=expected_alias)
 
-        self.assertEqual(expected_alias, result)
-        _bulk = mock.create_autospec(mock_elasticsearch.bulk)
-        _bulk.assert_called_with(body=expected_data)
+        with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \
+                patch.object(mock_elasticsearch, 'bulk') as bulk:
+            result = self.es_proxy.delete_document(data=data, index=expected_alias)
+            self.assertEqual(expected_alias, result)
+            bulk.assert_called_with(body=expected_data)
 
     @patch('uuid.uuid4')
     def test_delete_feature_document(self, mock_uuid: MagicMock) -> None:
         mock_elasticsearch = self.es_proxy.elasticsearch
         new_index_name = 'test_indx'
         mock_uuid.return_value = new_index_name
-        _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias)
-        _get_alias.return_value = dict([(new_index_name, {})])
         expected_alias = 'feature_search_index'
         data = ['id1', 'id2']
 
         expected_data = [
-            {'delete': {'_index': new_index_name, '_id': 'id1', '_type': 'feature'}},
-            {'delete': {'_index': new_index_name, '_id': 'id2', '_type': 'feature'}}
+            {'delete': {'_index': new_index_name, '_id': 'id1'}},
+            {'delete': {'_index': new_index_name, '_id': 'id2'}}
         ]
-        result = self.es_proxy.delete_document(data=data, index=expected_alias)
 
-        self.assertEqual(expected_alias, result)
-        _bulk = mock.create_autospec(mock_elasticsearch.bulk)
-        _bulk.assert_called_with(body=expected_data)
+        with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \
+                patch.object(mock_elasticsearch, 'bulk') as bulk:
+            result = self.es_proxy.delete_document(data=data, index=expected_alias)
+            self.assertEqual(expected_alias, result)
+            bulk.assert_called_with(body=expected_data)
 
     def test_get_instance_string(self) -> None:
         result = self.es_proxy._get_instance('column', 'value')