From 164b9560801d763fb54aaed5968a1f32b5936221 Mon Sep 17 00:00:00 2001 From: Kristen Armes <6732445+kristenarmes@users.noreply.github.com> Date: Thu, 4 Apr 2024 15:03:29 -0700 Subject: [PATCH] chore: Add py3.10 to compatibility matrix across all services except for metadata (#2242) * Add py3.10 to compatibility matrix Signed-off-by: Kristen Armes * Fix search tests and upgrade pyarrow Signed-off-by: Kristen Armes * One more search test fix Signed-off-by: Kristen Armes * lint Signed-off-by: Kristen Armes * Take 3.10 out for metadata, currently blocked by amundsengremlin related tests Signed-off-by: Kristen Armes --------- Signed-off-by: Kristen Armes --- .github/workflows/common_pull_request.yml | 2 +- .../workflows/databuilder_pull_request.yml | 2 +- .github/workflows/frontend_pull_request.yml | 2 +- .github/workflows/noop_pull_request.yml | 2 +- .github/workflows/search_pull_request.yml | 2 +- common/setup.py | 1 + databuilder/setup.py | 1 + frontend/setup.py | 3 +- search/setup.py | 1 + search/tests/unit/proxy/test_elasticsearch.py | 81 ++++++++----------- 10 files changed, 45 insertions(+), 52 deletions(-) diff --git a/.github/workflows/common_pull_request.yml b/.github/workflows/common_pull_request.yml index 3002c0d6a4..79afb79cfc 100644 --- a/.github/workflows/common_pull_request.yml +++ b/.github/workflows/common_pull_request.yml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.8.x', '3.9.x'] + python-version: ['3.8.x', '3.9.x', '3.10.x'] steps: - name: Checkout uses: actions/checkout@v1 diff --git a/.github/workflows/databuilder_pull_request.yml b/.github/workflows/databuilder_pull_request.yml index 93b1a748d7..715eb16232 100644 --- a/.github/workflows/databuilder_pull_request.yml +++ b/.github/workflows/databuilder_pull_request.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.8.x', '3.9.x'] + python-version: ['3.8.x', '3.9.x', '3.10.x'] steps: - name: Checkout uses: actions/checkout@v1 diff --git a/.github/workflows/frontend_pull_request.yml b/.github/workflows/frontend_pull_request.yml index f2934d0308..bb7ae80a73 100644 --- a/.github/workflows/frontend_pull_request.yml +++ b/.github/workflows/frontend_pull_request.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.8.x', '3.9.x'] + python-version: ['3.8.x', '3.9.x', '3.10.x'] steps: - name: Checkout uses: actions/checkout@v1 diff --git a/.github/workflows/noop_pull_request.yml b/.github/workflows/noop_pull_request.yml index 0b3eba6645..222122d5f3 100644 --- a/.github/workflows/noop_pull_request.yml +++ b/.github/workflows/noop_pull_request.yml @@ -30,7 +30,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.8.x', '3.9.x'] + python-version: ['3.8.x', '3.9.x', '3.10.x'] steps: - name: Do nothing run: exit 0 diff --git a/.github/workflows/search_pull_request.yml b/.github/workflows/search_pull_request.yml index b9694582e7..f84af9291c 100644 --- a/.github/workflows/search_pull_request.yml +++ b/.github/workflows/search_pull_request.yml @@ -10,7 +10,7 @@ jobs: runs-on: ubuntu-22.04 strategy: matrix: - python-version: ['3.8.x', '3.9.x'] + python-version: ['3.8.x', '3.9.x', '3.10.x'] steps: - name: Checkout uses: actions/checkout@v1 diff --git a/common/setup.py b/common/setup.py index 1e0f578e0a..a864d57340 100644 --- a/common/setup.py +++ b/common/setup.py @@ -51,5 +51,6 @@ classifiers=[ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], ) diff --git a/databuilder/setup.py b/databuilder/setup.py index 7e26d082c2..805380cb92 100644 --- a/databuilder/setup.py +++ b/databuilder/setup.py @@ -142,5 +142,6 @@ classifiers=[ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], ) diff --git a/frontend/setup.py b/frontend/setup.py index 0d3f67a0bb..6f415745e6 100644 --- a/frontend/setup.py +++ b/frontend/setup.py @@ -50,7 +50,7 @@ def build_js() -> None: jira = ['jira==3.0.1'] asana = ['asana==0.10.3'] oidc = ['flaskoidc>=1.0.0'] -pyarrow = ['pyarrow==3.0.0'] +pyarrow = ['pyarrow==15.0.0'] bigquery_preview = ['google-cloud-bigquery>=2.13.1,<3.0.0', 'flatten-dict==0.3.0'] all_deps = requirements + requirements_common + requirements_dev + oidc + pyarrow + bigquery_preview + jira + asana @@ -83,5 +83,6 @@ def build_js() -> None: classifiers=[ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], ) diff --git a/search/setup.py b/search/setup.py index 934e82b046..b456e4b323 100644 --- a/search/setup.py +++ b/search/setup.py @@ -44,5 +44,6 @@ classifiers=[ 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', + 'Programming Language :: Python :: 3.10', ], ) diff --git a/search/tests/unit/proxy/test_elasticsearch.py b/search/tests/unit/proxy/test_elasticsearch.py index ab6847d087..e9158c9461 100644 --- a/search/tests/unit/proxy/test_elasticsearch.py +++ b/search/tests/unit/proxy/test_elasticsearch.py @@ -5,7 +5,6 @@ from typing import ( # noqa: F401 Any, Iterable, List, Optional, ) -from unittest import mock from unittest.mock import MagicMock, patch from amundsen_common.models.api import health_check @@ -548,7 +547,6 @@ def test_create_document(self, mock_uuid: MagicMock) -> None: { 'index': { '_index': new_index_name, - '_type': 'table', '_id': 'snowflake://blue.test_schema/bank_accounts' } }, @@ -569,11 +567,11 @@ def test_create_document(self, mock_uuid: MagicMock) -> None: 'total_usage': 0, 'programmatic_descriptions': None, 'schema_description': 'schema description 1', + 'resource_type': 'table', }, { 'index': { '_index': new_index_name, - '_type': 'table', '_id': 'snowflake://blue.test_schema/bitcoin_wallets' } }, @@ -593,20 +591,17 @@ def test_create_document(self, mock_uuid: MagicMock) -> None: 'badges': [], 'total_usage': 0, 'schema_description': 'schema description 2', - 'programmatic_descriptions': ["test"] + 'programmatic_descriptions': ["test"], + 'resource_type': 'table', } ] - _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias) - _get_alias.return_value = dict([(new_index_name, {})]) - - _bulk = mock.create_autospec(mock_elasticsearch.indices.get_alias) - _bulk.return_value = {'errors': False} - expected_alias = 'table_search_index' - result = self.es_proxy.create_document(data=start_data, index=expected_alias) - self.assertEqual(expected_alias, result) - _bulk.assert_called_with(body=expected_data) + with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \ + patch.object(mock_elasticsearch, 'bulk', return_value={'errors': False}) as bulk: + result = self.es_proxy.create_document(data=start_data, index=expected_alias) + self.assertEqual(expected_alias, result) + bulk.assert_called_with(body=expected_data) def test_update_document_with_no_data(self) -> None: expected = '' @@ -617,8 +612,6 @@ def test_update_document_with_no_data(self) -> None: def test_update_document(self, mock_uuid: MagicMock) -> None: mock_elasticsearch = self.es_proxy.elasticsearch new_index_name = 'tester_index_name' - _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias) - _get_alias.return_value = dict([(new_index_name, {})]) mock_uuid.return_value = new_index_name table_key = 'snowflake://blue.test_schema/bitcoin_wallets' expected_alias = 'table_search_index' @@ -633,7 +626,6 @@ def test_update_document(self, mock_uuid: MagicMock) -> None: { 'update': { '_index': new_index_name, - '_type': 'table', '_id': table_key } }, @@ -655,75 +647,72 @@ def test_update_document(self, mock_uuid: MagicMock) -> None: 'total_usage': 0, 'programmatic_descriptions': None, 'schema_description': 'schema description 1', + 'resource_type': 'table', } } ] - result = self.es_proxy.update_document(data=data, index=expected_alias) - self.assertEqual(expected_alias, result) - _bulk = mock.create_autospec(mock_elasticsearch.bulk) - _bulk.assert_called_with(body=expected_data) + with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \ + patch.object(mock_elasticsearch, 'bulk') as bulk: + result = self.es_proxy.update_document(data=data, index=expected_alias) + self.assertEqual(expected_alias, result) + bulk.assert_called_with(body=expected_data) @patch('uuid.uuid4') def test_delete_table_document(self, mock_uuid: MagicMock) -> None: mock_elasticsearch = self.es_proxy.elasticsearch new_index_name = 'tester_index_name' mock_uuid.return_value = new_index_name - - _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias) - _get_alias.return_value = dict([(new_index_name, {})]) - expected_alias = 'table_search_index' data = ['id1', 'id2'] expected_data = [ - {'delete': {'_index': new_index_name, '_id': 'id1', '_type': 'table'}}, - {'delete': {'_index': new_index_name, '_id': 'id2', '_type': 'table'}} + {'delete': {'_index': new_index_name, '_id': 'id1'}}, + {'delete': {'_index': new_index_name, '_id': 'id2'}} ] - result = self.es_proxy.delete_document(data=data, index=expected_alias) - self.assertEqual(expected_alias, result) - _bulk = mock.create_autospec(mock_elasticsearch.bulk) - _bulk.assert_called_with(body=expected_data) + with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \ + patch.object(mock_elasticsearch, 'bulk') as bulk: + result = self.es_proxy.delete_document(data=data, index=expected_alias) + self.assertEqual(expected_alias, result) + bulk.assert_called_with(body=expected_data) @patch('uuid.uuid4') def test_delete_user_document(self, mock_uuid: MagicMock) -> None: mock_elasticsearch = self.es_proxy.elasticsearch new_index_name = 'tester_index_name' mock_uuid.return_value = new_index_name - _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias) - _get_alias.return_value = dict([(new_index_name, {})]) expected_alias = 'user_search_index' data = ['id1', 'id2'] expected_data = [ - {'delete': {'_index': new_index_name, '_id': 'id1', '_type': 'user'}}, - {'delete': {'_index': new_index_name, '_id': 'id2', '_type': 'user'}} + {'delete': {'_index': new_index_name, '_id': 'id1'}}, + {'delete': {'_index': new_index_name, '_id': 'id2'}} ] - result = self.es_proxy.delete_document(data=data, index=expected_alias) - self.assertEqual(expected_alias, result) - _bulk = mock.create_autospec(mock_elasticsearch.bulk) - _bulk.assert_called_with(body=expected_data) + with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \ + patch.object(mock_elasticsearch, 'bulk') as bulk: + result = self.es_proxy.delete_document(data=data, index=expected_alias) + self.assertEqual(expected_alias, result) + bulk.assert_called_with(body=expected_data) @patch('uuid.uuid4') def test_delete_feature_document(self, mock_uuid: MagicMock) -> None: mock_elasticsearch = self.es_proxy.elasticsearch new_index_name = 'test_indx' mock_uuid.return_value = new_index_name - _get_alias = mock.create_autospec(mock_elasticsearch.indices.get_alias) - _get_alias.return_value = dict([(new_index_name, {})]) expected_alias = 'feature_search_index' data = ['id1', 'id2'] expected_data = [ - {'delete': {'_index': new_index_name, '_id': 'id1', '_type': 'feature'}}, - {'delete': {'_index': new_index_name, '_id': 'id2', '_type': 'feature'}} + {'delete': {'_index': new_index_name, '_id': 'id1'}}, + {'delete': {'_index': new_index_name, '_id': 'id2'}} ] - result = self.es_proxy.delete_document(data=data, index=expected_alias) - self.assertEqual(expected_alias, result) - _bulk = mock.create_autospec(mock_elasticsearch.bulk) - _bulk.assert_called_with(body=expected_data) + with patch.object(mock_elasticsearch.indices, 'get_alias', return_value=dict([(new_index_name, {})])), \ + patch.object(mock_elasticsearch, 'bulk') as bulk: + result = self.es_proxy.delete_document(data=data, index=expected_alias) + self.assertEqual(expected_alias, result) + bulk.assert_called_with(body=expected_data) def test_get_instance_string(self) -> None: result = self.es_proxy._get_instance('column', 'value')