From 29136cd6003ceb83c4abbf722cfe79251487cf28 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Thu, 19 Sep 2024 13:23:48 -0600 Subject: [PATCH 01/15] Add "expose primary keys" to create_postgis. --- .../engines/geoserver_engine.py | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/tethys_dataset_services/engines/geoserver_engine.py b/tethys_dataset_services/engines/geoserver_engine.py index b6b6515..6b39803 100644 --- a/tethys_dataset_services/engines/geoserver_engine.py +++ b/tethys_dataset_services/engines/geoserver_engine.py @@ -1250,7 +1250,7 @@ def link_sqlalchemy_db_to_geoserver(self, store_id, sqlalchemy_engine, max_conne def create_postgis_store(self, store_id, host, port, database, username, password, max_connections=5, max_connection_idle_time=30, evictor_run_periodicity=30, validate_connections=True, - debug=False): + expose_primary_keys=False, debug=False): """ Use this method to link an existing PostGIS database to GeoServer as a feature store. Note that this method only works for data in vector formats. @@ -1265,6 +1265,7 @@ def create_postgis_store(self, store_id, host, port, database, username, passwor max_connection_idle_time (int, optional): Number of seconds a connections can stay idle before the evictor considers closing it. Defaults to 30 seconds. evictor_run_periodicity (int, optional): Number of seconds between idle connection evictor runs. Defaults to 30 seconds. validate_connections (bool, optional): Test connections before using. Defaults to True. + expose_primary_keys (bool, optional): debug (bool, optional): Pretty print the response dictionary to the console for debugging. Defaults to False. Returns: @@ -1283,24 +1284,24 @@ def create_postgis_store(self, store_id, host, port, database, username, passwor workspace = self.catalog.get_default_workspace().name # Create the store - xml = """ + xml = f""" - {0} + {name} - {1} - {2} - {3} - {4} - {5} + {host} + {port} + {database} + {username} + {password} postgis - {6} - {7} - {8} - {9} + {max_connections} + {max_connection_idle_time} + {evictor_run_periodicity} + {str(validate_connections).lower()} + {str(expose_primary_keys).lower()} - """.format(name, host, port, database, username, password, max_connections, max_connection_idle_time, - evictor_run_periodicity, str(validate_connections).lower()) + """ # Prepare headers headers = { From dc28726e8d8f73852395e69163a0b129cbf38960 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:05:25 -0600 Subject: [PATCH 02/15] Fix tests --- .github/workflows/e2e_tests.yml | 4 ++-- .github/workflows/unit_tests.yml | 10 ++++------ tests/unit_tests/test_geoserver_engine.py | 8 ++++---- tox.ini | 6 ++---- 4 files changed, 12 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index e1aca12..11f3d30 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -52,9 +52,9 @@ jobs: ports: - 5432:5432 steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.py }} for test - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} - name: Install dependencies diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 7bc2fa5..8bfa665 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -19,15 +19,13 @@ jobs: - windows - macos py: + - "3.12" - "3.11" - "3.10" - - "3.9" - - "3.8" - - "3.7" steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.py }} for test - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.py }} - name: Install dependencies @@ -38,7 +36,7 @@ jobs: - name: Run tests run: tox - name: Coveralls - if: matrix.os == 'ubuntu' && matrix.py == 3.10 + if: matrix.os == 'ubuntu' && matrix.py == 3.12 run: coveralls --service=github env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/tests/unit_tests/test_geoserver_engine.py b/tests/unit_tests/test_geoserver_engine.py index 77c976c..a17ea89 100644 --- a/tests/unit_tests/test_geoserver_engine.py +++ b/tests/unit_tests/test_geoserver_engine.py @@ -210,7 +210,7 @@ def test_list_resources(self, mock_catalog): for n in self.resource_names: self.assertIn(n, result) - mc.get_resources.called_with(store=None, workspace=None) + mc.get_resources.assert_called_with(store=None, workspace=None) @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog') def test_list_resources_with_properties(self, mock_catalog): @@ -244,7 +244,7 @@ def test_list_resources_with_properties(self, mock_catalog): self.assertIn('store', r) self.assertEqual(self.store_name, r['store']) - mc.get_resources.called_with(store=None, workspace=None) + mc.get_resources.assert_called_with(store=None, workspace=None) @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog') def test_list_resources_ambiguous_error(self, mock_catalog): @@ -260,7 +260,7 @@ def test_list_resources_ambiguous_error(self, mock_catalog): # Success self.assertFalse(response['success']) - mc.get_resources.called_with(store=None, workspace=None) + mc.get_resources.assert_called_with(store=None, workspace=None) @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog') def test_list_resources_multiple_stores_error(self, mock_catalog): @@ -277,7 +277,7 @@ def test_list_resources_multiple_stores_error(self, mock_catalog): self.assertFalse(response['success']) self.assertIn('Multiple stores found named', response['error']) - mc.get_resources.called_with(store=None, workspace=None) + mc.get_resources.assert_called_with(store=None, workspace=None) @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog') def test_list_layers(self, mock_catalog): diff --git a/tox.ini b/tox.ini index 8e4d288..86d262a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,14 +1,12 @@ [tox] isolated_build = True -envlist = py37, py38, py39, py310, py311, flake8, clean +envlist = py310, py311, py12, flake8, clean [gh-actions] python = - 3.7: py37 - 3.8: py38 - 3.9: py39 3.10: py310 3.11: py311 + 3.12: py312 [testenv] deps = From 338d6d0f3669e6d5ec09ad992b7e5bbaad3f8163 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:13:25 -0600 Subject: [PATCH 03/15] Fix tests --- tests/unit_tests/test_geoserver_engine.py | 61 +++++++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/tests/unit_tests/test_geoserver_engine.py b/tests/unit_tests/test_geoserver_engine.py index a17ea89..11c5f80 100644 --- a/tests/unit_tests/test_geoserver_engine.py +++ b/tests/unit_tests/test_geoserver_engine.py @@ -210,7 +210,7 @@ def test_list_resources(self, mock_catalog): for n in self.resource_names: self.assertIn(n, result) - mc.get_resources.assert_called_with(store=None, workspace=None) + mc.get_resources.assert_called_with(stores=None, workspaces=None) @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog') def test_list_resources_with_properties(self, mock_catalog): @@ -244,7 +244,7 @@ def test_list_resources_with_properties(self, mock_catalog): self.assertIn('store', r) self.assertEqual(self.store_name, r['store']) - mc.get_resources.assert_called_with(store=None, workspace=None) + mc.get_resources.assert_called_with(stores=None, workspaces=None) @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog') def test_list_resources_ambiguous_error(self, mock_catalog): @@ -260,7 +260,7 @@ def test_list_resources_ambiguous_error(self, mock_catalog): # Success self.assertFalse(response['success']) - mc.get_resources.assert_called_with(store=None, workspace=None) + mc.get_resources.assert_called_with(stores=None, workspaces=None) @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog') def test_list_resources_multiple_stores_error(self, mock_catalog): @@ -277,7 +277,7 @@ def test_list_resources_multiple_stores_error(self, mock_catalog): self.assertFalse(response['success']) self.assertIn('Multiple stores found named', response['error']) - mc.get_resources.assert_called_with(store=None, workspace=None) + mc.get_resources.assert_called_with(stores=None, workspaces=None) @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog') def test_list_layers(self, mock_catalog): @@ -3708,6 +3708,7 @@ def test_create_postgis_store_validate_connection(self, mock_post, _): {7} {8} true + false """.format('foo', host, port, database, username, password, max_connections, max_connection_idle_time, @@ -3756,6 +3757,7 @@ def test_create_postgis_store_validate_connection_false(self, mock_workspace, mo {7} {8} false + false """.format('foo', host, port, database, username, password, max_connections, max_connection_idle_time, @@ -3774,6 +3776,56 @@ def test_create_postgis_store_validate_connection_false(self, mock_workspace, mo max_connection_idle_time, evictor_run_periodicity, validate_connections=False) mock_post.assert_called_with(url=rest_endpoint, data=xml, headers=expected_headers, auth=self.auth) + @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerSpatialDatasetEngine.get_store') + @mock.patch('tethys_dataset_services.engines.geoserver_engine.requests.post') + @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerCatalog.get_default_workspace') + def test_create_postgis_store_expose_primary_keys_true(self, mock_workspace, mock_post, _): + mock_post.return_value = MockResponse(201) + store_id = 'foo' + mock_workspace().name = self.workspace_name + host = 'localhost' + port = '5432' + database = 'foo_db' + username = 'user' + password = 'pass' + max_connections = 10 + max_connection_idle_time = 40 + evictor_run_periodicity = 60 + + xml = """ + + {0} + + {1} + {2} + {3} + {4} + {5} + postgis + {6} + {7} + {8} + false + true + + + """.format('foo', host, port, database, username, password, max_connections, max_connection_idle_time, + evictor_run_periodicity) + + expected_headers = { + "Content-type": "text/xml", + "Accept": "application/xml" + } + + rest_endpoint = '{endpoint}workspaces/{workspace}/datastores'.format( + endpoint=self.endpoint, + workspace=self.workspace_name + ) + self.engine.create_postgis_store(store_id, host, port, database, username, password, max_connections, + max_connection_idle_time, evictor_run_periodicity, validate_connections=False, + expose_primary_keys=True) + mock_post.assert_called_with(url=rest_endpoint, data=xml, headers=expected_headers, auth=self.auth) + @mock.patch('tethys_dataset_services.engines.geoserver_engine.GeoServerSpatialDatasetEngine.get_store') @mock.patch('tethys_dataset_services.engines.geoserver_engine.log') @mock.patch('tethys_dataset_services.engines.geoserver_engine.requests.post') @@ -3803,6 +3855,7 @@ def test_create_postgis_store_not_201(self, mock_post, mock_logger, _): {7} {8} true + false """.format('foo', host, port, database, username, password, max_connections, max_connection_idle_time, From 9b918abcca4a876e8c86ce940ad980cb03821e2f Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:16:58 -0600 Subject: [PATCH 04/15] Add back 3.9 tests --- .github/workflows/e2e_tests.yml | 2 +- .github/workflows/unit_tests.yml | 1 + tox.ini | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 11f3d30..d9a01be 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -15,7 +15,7 @@ jobs: fail-fast: false matrix: py: - - "3.10" + - "3.12" postgis: - "9.6-2.5-alpine" - "10-2.5-alpine" diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 8bfa665..3d00fd1 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -22,6 +22,7 @@ jobs: - "3.12" - "3.11" - "3.10" + - "3.9" steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.py }} for test diff --git a/tox.ini b/tox.ini index 86d262a..366bd6d 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,10 @@ [tox] isolated_build = True -envlist = py310, py311, py12, flake8, clean +envlist = py39, py310, py311, py12, flake8, clean [gh-actions] python = + 3.9: py39 3.10: py310 3.11: py311 3.12: py312 From 214f655aa83be2a3a081b171ff35a02e61f6f345 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:24:35 -0600 Subject: [PATCH 05/15] Add 3.13 to unit test matrix --- .github/workflows/unit_tests.yml | 1 + tox.ini | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 3d00fd1..a514526 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -19,6 +19,7 @@ jobs: - windows - macos py: + - "3.13" - "3.12" - "3.11" - "3.10" diff --git a/tox.ini b/tox.ini index 366bd6d..1d7fa02 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] isolated_build = True -envlist = py39, py310, py311, py12, flake8, clean +envlist = py39, py310, py311, py312, py313, flake8, clean [gh-actions] python = @@ -8,6 +8,7 @@ python = 3.10: py310 3.11: py311 3.12: py312 + 3.13: py313 [testenv] deps = From 32935cc7f080b73953e34cf5127b58f54a6672aa Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:30:12 -0600 Subject: [PATCH 06/15] Enable postgis 3.4 tests to see what breaks --- .github/workflows/e2e_tests.yml | 11 ++++++----- .github/workflows/unit_tests.yml | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index d9a01be..3da9c84 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -21,11 +21,12 @@ jobs: - "10-2.5-alpine" - "11-2.5-alpine" - "12-2.5-alpine" - # Not working on PostGIS 3.2 yet - # - "9.6-3.2-alpine" - # - "10-3.2-alpine" - # - "11-3.2-alpine" - # - "12-3.2-alpine" + # Not working on PostGIS 3.4 yet + - "12-3.4-alpine" + - "13-3.4-alpine" + - "14-3.4-alpine" + - "15-3.4-alpine" + - "16-3.4-alpine" services: geoserver: image: tethysplatform/geoserver diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index a514526..3ce19fe 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -19,7 +19,7 @@ jobs: - windows - macos py: - - "3.13" + # - "3.13" - "3.12" - "3.11" - "3.10" From 3c0c165d78725f56d96492949b11d432de75d8f2 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:35:37 -0600 Subject: [PATCH 07/15] Create postgis_raster in e2e tests --- .github/workflows/e2e_tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 3da9c84..e4db625 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -67,6 +67,7 @@ jobs: run: | PGPASSWORD=mysecretpassword psql -U postgres -h localhost -c "CREATE DATABASE tds_tests WITH OWNER postgres;" PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis;" + PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;" sleep 10 - name: Run tests run: tox -e e2e_gs_tests From 5c058f508774a0a28155184b7623db761c203cb6 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:39:10 -0600 Subject: [PATCH 08/15] Only add postgis_raster to 3.4 runs --- .github/workflows/e2e_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index e4db625..8988bb3 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -67,7 +67,7 @@ jobs: run: | PGPASSWORD=mysecretpassword psql -U postgres -h localhost -c "CREATE DATABASE tds_tests WITH OWNER postgres;" PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis;" - PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;" + if [[ ${{ matrix.py }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;" fi sleep 10 - name: Run tests run: tox -e e2e_gs_tests From e78698bcd533b7df9ec502ad30ff4402be98544d Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:42:36 -0600 Subject: [PATCH 09/15] semicolon --- .github/workflows/e2e_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 8988bb3..9db8be7 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -67,7 +67,7 @@ jobs: run: | PGPASSWORD=mysecretpassword psql -U postgres -h localhost -c "CREATE DATABASE tds_tests WITH OWNER postgres;" PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis;" - if [[ ${{ matrix.py }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;" fi + if [[ ${{ matrix.py }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;"; fi sleep 10 - name: Run tests run: tox -e e2e_gs_tests From b3a59af5efc6df24019c6c0ca9447c0abd02c51c Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:44:35 -0600 Subject: [PATCH 10/15] matrix.postgis not matrix.py --- .github/workflows/e2e_tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 9db8be7..9e52d2e 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -67,7 +67,7 @@ jobs: run: | PGPASSWORD=mysecretpassword psql -U postgres -h localhost -c "CREATE DATABASE tds_tests WITH OWNER postgres;" PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis;" - if [[ ${{ matrix.py }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;"; fi + if [[ ${{ matrix.postgis }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;"; fi sleep 10 - name: Run tests run: tox -e e2e_gs_tests From 8fe69e12550da943003b3f02fa60b237d57dcdf2 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 16:56:54 -0600 Subject: [PATCH 11/15] Enable debug on failing tests --- tests/e2e_tests/geoserver_engine_e2e_tests.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/tests/e2e_tests/geoserver_engine_e2e_tests.py b/tests/e2e_tests/geoserver_engine_e2e_tests.py index 84bda01..7b95ed7 100644 --- a/tests/e2e_tests/geoserver_engine_e2e_tests.py +++ b/tests/e2e_tests/geoserver_engine_e2e_tests.py @@ -989,10 +989,12 @@ def test_link_and_add_table(self): # Execute response = self.geoserver_engine.create_layer_from_postgis_store( store_id=store_id, - table=self.pg_table_name + table=self.pg_table_name, + debug=True ) # Check for success response + print(response) self.assertTrue(response['success']) # TEST list_stores @@ -1136,8 +1138,10 @@ def test_create_sql_view_layer(self): # Create layer from postgis store response = self.geoserver_engine.create_layer_from_postgis_store( store_id=store_id, - table=self.pg_table_name + table=self.pg_table_name, + debug=True ) + print(response) self.assertTrue(response['success']) # Pause to let GeoServer catch up before continuing From b6c44e03015f69bc511f2d6f95c74b7382e98cb1 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 17:07:02 -0600 Subject: [PATCH 12/15] Hack to force GeoServer to use full scan for extents. --- .github/workflows/e2e_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 9e52d2e..4f2659a 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -68,6 +68,9 @@ jobs: PGPASSWORD=mysecretpassword psql -U postgres -h localhost -c "CREATE DATABASE tds_tests WITH OWNER postgres;" PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis;" if [[ ${{ matrix.postgis }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;"; fi + # The next two are workarounds for issues with PostGIS calculating extents in PostGIS 3.4 (see: https://gis.stackexchange.com/questions/75536/geoserver-not-reading-bounding-box-from-postgis-geometry-correctly) + if [[ ${{ matrix.postgis }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "create or replace function st_estimated_extent(text,text) returns box2d as 'select null::box2d' language sql;"; fi + if [[ ${{ matrix.postgis }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "create or replace function st_estimated_extent(text,text,text) returns box2d as 'select null::box2d' language sql;"; fi sleep 10 - name: Run tests run: tox -e e2e_gs_tests From 4bbf9487b73dc25ae0760abd3d50207c25ca267e Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 17:17:27 -0600 Subject: [PATCH 13/15] Change test data to cover larger geographic scope to see if it solves issue with extent calculation --- .github/workflows/e2e_tests.yml | 3 --- tests/e2e_tests/geoserver_engine_e2e_tests.py | 3 ++- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 4f2659a..9e52d2e 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -68,9 +68,6 @@ jobs: PGPASSWORD=mysecretpassword psql -U postgres -h localhost -c "CREATE DATABASE tds_tests WITH OWNER postgres;" PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis;" if [[ ${{ matrix.postgis }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "CREATE EXTENSION postgis_raster;"; fi - # The next two are workarounds for issues with PostGIS calculating extents in PostGIS 3.4 (see: https://gis.stackexchange.com/questions/75536/geoserver-not-reading-bounding-box-from-postgis-geometry-correctly) - if [[ ${{ matrix.postgis }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "create or replace function st_estimated_extent(text,text) returns box2d as 'select null::box2d' language sql;"; fi - if [[ ${{ matrix.postgis }} == *"3.4-alpine"* ]]; then PGPASSWORD=mysecretpassword psql -U postgres -h localhost -d tds_tests -c "create or replace function st_estimated_extent(text,text,text) returns box2d as 'select null::box2d' language sql;"; fi sleep 10 - name: Run tests run: tox -e e2e_gs_tests diff --git a/tests/e2e_tests/geoserver_engine_e2e_tests.py b/tests/e2e_tests/geoserver_engine_e2e_tests.py index 7b95ed7..1ebb471 100644 --- a/tests/e2e_tests/geoserver_engine_e2e_tests.py +++ b/tests/e2e_tests/geoserver_engine_e2e_tests.py @@ -131,7 +131,8 @@ def setup_postgis_table(self): insert_sql = "INSERT INTO {table} VALUES ({id}, '{name}', ST_GeomFromText('POINT({lon} {lat})', 4326));" rows = [ {"id": 1, "name": "Aquaveo", "lat": 40.276039, "lon": -111.651120}, - {"id": 2, "name": "BYU", "lat": 40.252335, "lon": -111.649326}, + {"id": 2, "name": "Lynker", "lat": 39.111534, "lon": -77.556859}, + {"id": 3, "name": "CHL", "lat": 32.299343, "lon": -90.866044}, ] for r in rows: From 42bbf72853a1c64d661e5b76c03985ec1450e7a4 Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 17:31:06 -0600 Subject: [PATCH 14/15] Disable failing checks in 3.4 --- tests/e2e_tests/geoserver_engine_e2e_tests.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/e2e_tests/geoserver_engine_e2e_tests.py b/tests/e2e_tests/geoserver_engine_e2e_tests.py index 1ebb471..8e64f5c 100644 --- a/tests/e2e_tests/geoserver_engine_e2e_tests.py +++ b/tests/e2e_tests/geoserver_engine_e2e_tests.py @@ -995,8 +995,8 @@ def test_link_and_add_table(self): ) # Check for success response - print(response) - self.assertTrue(response['success']) + # TODO: returns an error in PostGIS 3.4: Internal Server Error(500): :java.io.IOException: Error occured calculating bounds for points + # self.assertTrue(response['success']) # TEST list_stores @@ -1142,8 +1142,9 @@ def test_create_sql_view_layer(self): table=self.pg_table_name, debug=True ) - print(response) - self.assertTrue(response['success']) + + # TODO: returns an error in PostGIS 3.4: Internal Server Error(500): :java.io.IOException: Error occured calculating bounds for points + # self.assertTrue(response['success']) # Pause to let GeoServer catch up before continuing sleep(5) From 066a50798445e0b57d5b6387e35f68e2eb24d09d Mon Sep 17 00:00:00 2001 From: Nathan Swain Date: Mon, 23 Sep 2024 17:56:24 -0600 Subject: [PATCH 15/15] Disable postgis 3.4 tests for now --- .github/workflows/e2e_tests.yml | 10 +++++----- tests/e2e_tests/geoserver_engine_e2e_tests.py | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e_tests.yml b/.github/workflows/e2e_tests.yml index 9e52d2e..c62f8ac 100644 --- a/.github/workflows/e2e_tests.yml +++ b/.github/workflows/e2e_tests.yml @@ -22,11 +22,11 @@ jobs: - "11-2.5-alpine" - "12-2.5-alpine" # Not working on PostGIS 3.4 yet - - "12-3.4-alpine" - - "13-3.4-alpine" - - "14-3.4-alpine" - - "15-3.4-alpine" - - "16-3.4-alpine" + # - "12-3.4-alpine" + # - "13-3.4-alpine" + # - "14-3.4-alpine" + # - "15-3.4-alpine" + # - "16-3.4-alpine" services: geoserver: image: tethysplatform/geoserver diff --git a/tests/e2e_tests/geoserver_engine_e2e_tests.py b/tests/e2e_tests/geoserver_engine_e2e_tests.py index 8e64f5c..177ff37 100644 --- a/tests/e2e_tests/geoserver_engine_e2e_tests.py +++ b/tests/e2e_tests/geoserver_engine_e2e_tests.py @@ -996,7 +996,7 @@ def test_link_and_add_table(self): # Check for success response # TODO: returns an error in PostGIS 3.4: Internal Server Error(500): :java.io.IOException: Error occured calculating bounds for points - # self.assertTrue(response['success']) + self.assertTrue(response['success']) # TEST list_stores @@ -1144,7 +1144,7 @@ def test_create_sql_view_layer(self): ) # TODO: returns an error in PostGIS 3.4: Internal Server Error(500): :java.io.IOException: Error occured calculating bounds for points - # self.assertTrue(response['success']) + self.assertTrue(response['success']) # Pause to let GeoServer catch up before continuing sleep(5)