diff --git a/nominatim/api/v1/format.py b/nominatim/api/v1/format.py index ad635e39cd..1e37b4c7a6 100644 --- a/nominatim/api/v1/format.py +++ b/nominatim/api/v1/format.py @@ -7,8 +7,9 @@ """ Output formatters for API version v1. """ -from typing import Mapping, Any +from typing import List, Dict, Mapping, Any import collections +import datetime as dt import nominatim.api as napi from nominatim.api.result_formatting import FormatDispatcher @@ -16,6 +17,10 @@ from nominatim.api.v1 import format_json, format_xml from nominatim.utils.json_writer import JsonWriter +class RawDataList(List[Dict[str, Any]]): + """ Data type for formatting raw data lists 'as is' in json. + """ + dispatch = FormatDispatcher() @dispatch.format_func(napi.StatusResult, 'text') @@ -232,3 +237,20 @@ def _format_search_jsonv2(results: napi.SearchResults, options: Mapping[str, Any]) -> str: return format_json.format_base_json(results, options, False, class_label='category') + +@dispatch.format_func(RawDataList, 'json') +def _format_raw_data_json(results: RawDataList, _: Mapping[str, Any]) -> str: + out = JsonWriter() + out.start_array() + for res in results: + out.start_object() + for k, v in res.items(): + if isinstance(v, dt.datetime): + out.keyval(k, v.isoformat(sep= ' ', timespec='seconds')) + else: + out.keyval(k, v) + out.end_object().next() + + out.end_array() + + return out() diff --git a/nominatim/api/v1/server_glue.py b/nominatim/api/v1/server_glue.py index 5b6efe5fc6..5ebdb55e96 100644 --- a/nominatim/api/v1/server_glue.py +++ b/nominatim/api/v1/server_glue.py @@ -15,11 +15,14 @@ import math from urllib.parse import urlencode +import sqlalchemy as sa + from nominatim.errors import UsageError from nominatim.config import Configuration import nominatim.api as napi import nominatim.api.logging as loglib from nominatim.api.v1.format import dispatch as formatting +from nominatim.api.v1.format import RawDataList from nominatim.api.v1 import helpers CONTENT_TYPE = { @@ -494,6 +497,58 @@ async def search_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A return params.build_response(output) +async def deletable_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any: + """ Server glue for /deletable endpoint. + This is a special endpoint that shows polygons that have been + deleted or are broken in the OSM data but are kept in the + Nominatim database to minimize disruption. + """ + fmt = params.parse_format(RawDataList, 'json') + + async with api.begin() as conn: + sql = sa.text(""" SELECT p.place_id, country_code, + name->'name' as name, i.* + FROM placex p, import_polygon_delete i + WHERE p.osm_id = i.osm_id AND p.osm_type = i.osm_type + AND p.class = i.class AND p.type = i.type + """) + results = RawDataList(r._asdict() for r in await conn.execute(sql)) + + return params.build_response(formatting.format_result(results, fmt, {})) + + +async def polygons_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> Any: + """ Server glue for /polygons endpoint. + This is a special endpoint that shows polygons that have changed + thier size but are kept in the Nominatim database with their + old area to minimize disruption. + """ + fmt = params.parse_format(RawDataList, 'json') + sql_params: Dict[str, Any] = { + 'days': params.get_int('days', -1), + 'cls': params.get('class') + } + reduced = params.get_bool('reduced', False) + + async with api.begin() as conn: + sql = sa.select(sa.text("""osm_type, osm_id, class, type, + name->'name' as name, + country_code, errormessage, updated"""))\ + .select_from(sa.text('import_polygon_error')) + if sql_params['days'] > 0: + sql = sql.where(sa.text("updated > 'now'::timestamp - make_interval(days => :days)")) + if reduced: + sql = sql.where(sa.text("errormessage like 'Area reduced%'")) + if sql_params['cls'] is not None: + sql = sql.where(sa.text("class = :cls")) + + sql = sql.order_by(sa.literal_column('updated').desc()).limit(1000) + + results = RawDataList(r._asdict() for r in await conn.execute(sql, sql_params)) + + return params.build_response(formatting.format_result(results, fmt, {})) + + EndpointFunc = Callable[[napi.NominatimAPIAsync, ASGIAdaptor], Any] ROUTES = [ @@ -501,5 +556,7 @@ async def search_endpoint(api: napi.NominatimAPIAsync, params: ASGIAdaptor) -> A ('details', details_endpoint), ('reverse', reverse_endpoint), ('lookup', lookup_endpoint), - ('search', search_endpoint) + ('search', search_endpoint), + ('deletable', deletable_endpoint), + ('polygons', polygons_endpoint), ] diff --git a/test/python/api/fake_adaptor.py b/test/python/api/fake_adaptor.py new file mode 100644 index 0000000000..1db8c72580 --- /dev/null +++ b/test/python/api/fake_adaptor.py @@ -0,0 +1,52 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Provides dummy implementations of ASGIAdaptor for testing. +""" +from collections import namedtuple + +import nominatim.api.v1.server_glue as glue +from nominatim.config import Configuration + +class FakeError(BaseException): + + def __init__(self, msg, status): + self.msg = msg + self.status = status + + def __str__(self): + return f'{self.status} -- {self.msg}' + +FakeResponse = namedtuple('FakeResponse', ['status', 'output', 'content_type']) + +class FakeAdaptor(glue.ASGIAdaptor): + + def __init__(self, params=None, headers=None, config=None): + self.params = params or {} + self.headers = headers or {} + self._config = config or Configuration(None) + + + def get(self, name, default=None): + return self.params.get(name, default) + + + def get_header(self, name, default=None): + return self.headers.get(name, default) + + + def error(self, msg, status=400): + return FakeError(msg, status) + + + def create_response(self, status, output): + return FakeResponse(status, output, self.content_type) + + + def config(self): + return self._config + diff --git a/test/python/api/test_api_deletable_v1.py b/test/python/api/test_api_deletable_v1.py new file mode 100644 index 0000000000..4c5d96b633 --- /dev/null +++ b/test/python/api/test_api_deletable_v1.py @@ -0,0 +1,67 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Tests for the deletable v1 API call. +""" +import json +from pathlib import Path + +import pytest +import pytest_asyncio + +import psycopg2.extras + +from fake_adaptor import FakeAdaptor, FakeError, FakeResponse + +import nominatim.api.v1.server_glue as glue +import nominatim.api as napi + +@pytest_asyncio.fixture +async def api(): + api = napi.NominatimAPIAsync(Path('/invalid')) + yield api + await api.close() + + +class TestDeletableEndPoint: + + @pytest.fixture(autouse=True) + def setup_deletable_table(self, temp_db_cursor, table_factory, temp_db_with_extensions): + psycopg2.extras.register_hstore(temp_db_cursor) + table_factory('import_polygon_delete', + definition='osm_id bigint, osm_type char(1), class text, type text', + content=[(345, 'N', 'boundary', 'administrative'), + (781, 'R', 'landuse', 'wood'), + (781, 'R', 'landcover', 'grass')]) + table_factory('placex', + definition="""place_id bigint, osm_id bigint, osm_type char(1), + class text, type text, name HSTORE, country_code char(2)""", + content=[(1, 345, 'N', 'boundary', 'administrative', {'old_name': 'Former'}, 'ab'), + (2, 781, 'R', 'landuse', 'wood', {'name': 'Wood'}, 'cd'), + (3, 781, 'R', 'landcover', 'grass', None, 'cd')]) + + + + @pytest.mark.asyncio + async def test_deletable(self, api): + a = FakeAdaptor() + + resp = await glue.deletable_endpoint(api, a) + results = json.loads(resp.output) + + results.sort(key=lambda r: r['place_id']) + + assert results == [{'place_id': 1, 'country_code': 'ab', 'name': None, + 'osm_id': 345, 'osm_type': 'N', + 'class': 'boundary', 'type': 'administrative'}, + {'place_id': 2, 'country_code': 'cd', 'name': 'Wood', + 'osm_id': 781, 'osm_type': 'R', + 'class': 'landuse', 'type': 'wood'}, + {'place_id': 3, 'country_code': 'cd', 'name': None, + 'osm_id': 781, 'osm_type': 'R', + 'class': 'landcover', 'type': 'grass'}] + diff --git a/test/python/api/test_api_polygons_v1.py b/test/python/api/test_api_polygons_v1.py new file mode 100644 index 0000000000..6842f791f7 --- /dev/null +++ b/test/python/api/test_api_polygons_v1.py @@ -0,0 +1,111 @@ +# SPDX-License-Identifier: GPL-3.0-or-later +# +# This file is part of Nominatim. (https://nominatim.org) +# +# Copyright (C) 2023 by the Nominatim developer community. +# For a full list of authors see the git log. +""" +Tests for the deletable v1 API call. +""" +import json +import datetime as dt +from pathlib import Path + +import pytest +import pytest_asyncio + +import psycopg2.extras + +from fake_adaptor import FakeAdaptor, FakeError, FakeResponse + +import nominatim.api.v1.server_glue as glue +import nominatim.api as napi + +@pytest_asyncio.fixture +async def api(): + api = napi.NominatimAPIAsync(Path('/invalid')) + yield api + await api.close() + + +class TestPolygonsEndPoint: + + @pytest.fixture(autouse=True) + def setup_deletable_table(self, temp_db_cursor, table_factory, temp_db_with_extensions): + psycopg2.extras.register_hstore(temp_db_cursor) + + self.now = dt.datetime.now() + self.recent = dt.datetime.now() - dt.timedelta(days=3) + + table_factory('import_polygon_error', + definition="""osm_id bigint, + osm_type character(1), + class text, + type text, + name hstore, + country_code character varying(2), + updated timestamp without time zone, + errormessage text, + prevgeometry geometry(Geometry,4326), + newgeometry geometry(Geometry,4326)""", + content=[(345, 'N', 'boundary', 'administrative', + {'name': 'Foo'}, 'xx', self.recent, + 'some text', None, None), + (781, 'R', 'landuse', 'wood', + None, 'ds', self.now, + 'Area reduced by lots', None, None)]) + + + @pytest.mark.asyncio + async def test_polygons_simple(self, api): + a = FakeAdaptor() + + resp = await glue.polygons_endpoint(api, a) + results = json.loads(resp.output) + + results.sort(key=lambda r: (r['osm_type'], r['osm_id'])) + + assert results == [{'osm_type': 'N', 'osm_id': 345, + 'class': 'boundary', 'type': 'administrative', + 'name': 'Foo', 'country_code': 'xx', + 'errormessage': 'some text', + 'updated': self.recent.isoformat(sep=' ', timespec='seconds')}, + {'osm_type': 'R', 'osm_id': 781, + 'class': 'landuse', 'type': 'wood', + 'name': None, 'country_code': 'ds', + 'errormessage': 'Area reduced by lots', + 'updated': self.now.isoformat(sep=' ', timespec='seconds')}] + + + @pytest.mark.asyncio + async def test_polygons_days(self, api): + a = FakeAdaptor() + a.params['days'] = '2' + + resp = await glue.polygons_endpoint(api, a) + results = json.loads(resp.output) + + assert [r['osm_id'] for r in results] == [781] + + + @pytest.mark.asyncio + async def test_polygons_class(self, api): + a = FakeAdaptor() + a.params['class'] = 'landuse' + + resp = await glue.polygons_endpoint(api, a) + results = json.loads(resp.output) + + assert [r['osm_id'] for r in results] == [781] + + + + @pytest.mark.asyncio + async def test_polygons_reduced(self, api): + a = FakeAdaptor() + a.params['reduced'] = '1' + + resp = await glue.polygons_endpoint(api, a) + results = json.loads(resp.output) + + assert [r['osm_id'] for r in results] == [781] diff --git a/test/python/api/test_server_glue_v1.py b/test/python/api/test_server_glue_v1.py index a731e72034..26e6517e53 100644 --- a/test/python/api/test_server_glue_v1.py +++ b/test/python/api/test_server_glue_v1.py @@ -7,56 +7,18 @@ """ Tests for the Python web frameworks adaptor, v1 API. """ -from collections import namedtuple import json import xml.etree.ElementTree as ET from pathlib import Path import pytest -from nominatim.config import Configuration +from fake_adaptor import FakeAdaptor, FakeError, FakeResponse + import nominatim.api.v1.server_glue as glue import nominatim.api as napi import nominatim.api.logging as loglib -class FakeError(BaseException): - - def __init__(self, msg, status): - self.msg = msg - self.status = status - - def __str__(self): - return f'{self.status} -- {self.msg}' - -FakeResponse = namedtuple('FakeResponse', ['status', 'output', 'content_type']) - -class FakeAdaptor(glue.ASGIAdaptor): - - def __init__(self, params=None, headers=None, config=None): - self.params = params or {} - self.headers = headers or {} - self._config = config or Configuration(None) - - - def get(self, name, default=None): - return self.params.get(name, default) - - - def get_header(self, name, default=None): - return self.headers.get(name, default) - - - def error(self, msg, status=400): - return FakeError(msg, status) - - - def create_response(self, status, output): - return FakeResponse(status, output, self.content_type) - - - def config(self): - return self._config - # ASGIAdaptor.get_int/bool()