From d2b46c9ae86765f302370b65aa5410581bdc49a9 Mon Sep 17 00:00:00 2001 From: Matt McFarland Date: Thu, 18 Jul 2024 12:40:01 -0400 Subject: [PATCH] Add support and tests for legacy mosaic tile routes (#234) * Add support and tests for legacy mosaic tile routes The tile route prefixes were reordered in recent versions of titiler.pgstac. Both versions of the routes are supported in PC and tests have been added to confirm. --- deployment/bin/lib | 18 ++-- pccommon/pccommon/config/collections.py | 5 +- pctiler/pctiler/endpoints/dependencies.py | 21 ++++ pctiler/pctiler/endpoints/item.py | 20 +--- pctiler/pctiler/endpoints/pg_mosaic.py | 98 +++++++++++++++++- pctiler/pctiler/main.py | 5 + pctiler/pctiler/middleware.py | 14 ++- pctiler/tests/endpoints/test_pg_mosaic.py | 116 ++++++++++++++++++++-- 8 files changed, 254 insertions(+), 43 deletions(-) create mode 100644 pctiler/pctiler/endpoints/dependencies.py diff --git a/deployment/bin/lib b/deployment/bin/lib index c2c7cbf5..858ec5c8 100755 --- a/deployment/bin/lib +++ b/deployment/bin/lib @@ -143,7 +143,8 @@ function disable_shared_access_keys() { --resource-group ${SAK_RESOURCE_GROUP} \ --allow-shared-key-access false \ --subscription ${ARM_SUBSCRIPTION_ID} \ - --output none + --output none \ + --only-show-errors if [ $? -ne 0 ]; then echo "!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!" @@ -170,7 +171,8 @@ function enable_shared_access_keys() { --resource-group ${SAK_RESOURCE_GROUP} \ --allow-shared-key-access true \ --subscription ${ARM_SUBSCRIPTION_ID} \ - --output none + --output none \ + --only-show-errors done sleep 10 @@ -185,7 +187,8 @@ function add_ip_to_firewalls() { -n "${KEY_VAULT_NAME}" \ --ip-address "$cidr" \ --subscription "${ARM_SUBSCRIPTION_ID}" \ - --output none + --output none \ + --only-show-errors # Also add the IP to the terraform state storage account for FW_STORAGE_ACCOUNT in "${!FW_STORAGE_ACCOUNTS[@]}"; do @@ -196,7 +199,8 @@ function add_ip_to_firewalls() { -n "${FW_STORAGE_ACCOUNT}" \ --ip-address "$cidr" \ --subscription "${ARM_SUBSCRIPTION_ID}" \ - --output none + --output none \ + --only-show-errors done sleep 10 @@ -211,7 +215,8 @@ function remove_ip_from_firewalls() { -n ${KEY_VAULT_NAME} \ --ip-address $cidr \ --subscription ${ARM_SUBSCRIPTION_ID} \ - --output none + --output none \ + --only-show-errors for FW_STORAGE_ACCOUNT in "${!FW_STORAGE_ACCOUNTS[@]}"; do FW_RESOURCE_GROUP=${FW_STORAGE_ACCOUNTS[$FW_STORAGE_ACCOUNT]} @@ -221,6 +226,7 @@ function remove_ip_from_firewalls() { -n ${FW_STORAGE_ACCOUNT} \ --ip-address $cidr \ --subscription ${ARM_SUBSCRIPTION_ID} \ - --output none + --output none \ + --only-show-errors done } diff --git a/pccommon/pccommon/config/collections.py b/pccommon/pccommon/config/collections.py index bcbc145b..a627f336 100644 --- a/pccommon/pccommon/config/collections.py +++ b/pccommon/pccommon/config/collections.py @@ -24,6 +24,7 @@ class CamelModel(BaseModel): # https://docs.pydantic.dev/latest/api/config/#pydantic.alias_generators.to_camel "alias_generator": camelize, "populate_by_name": True, + "use_enum_values": True, } @@ -267,7 +268,9 @@ class RenderOptions(CamelModel): name: str description: Optional[str] = None - type: Optional[RenderOptionType] = Field(default=RenderOptionType.raster_tile) + type: Optional[RenderOptionType] = Field( + default=RenderOptionType.raster_tile, validate_default=True + ) options: Optional[str] vector_options: Optional[VectorTileOptions] = None min_zoom: int diff --git a/pctiler/pctiler/endpoints/dependencies.py b/pctiler/pctiler/endpoints/dependencies.py new file mode 100644 index 00000000..bfd4559b --- /dev/null +++ b/pctiler/pctiler/endpoints/dependencies.py @@ -0,0 +1,21 @@ +import logging +from typing import Callable + +import fastapi +import starlette + +logger = logging.getLogger(__name__) + + +def get_endpoint_function( + router: fastapi.APIRouter, path: str, method: str +) -> Callable: + for route in router.routes: + match, _ = route.matches({"type": "http", "path": path, "method": method}) + if match == starlette.routing.Match.FULL: + # The abstract BaseRoute doesn't have a `.endpoint` attribute, + # but all of its subclasses do. + return route.endpoint # type: ignore [attr-defined] + + logger.warning(f"Could not find endpoint. method={method} path={path}") + raise fastapi.HTTPException(detail="Internal system error", status_code=500) diff --git a/pctiler/pctiler/endpoints/item.py b/pctiler/pctiler/endpoints/item.py index e82a09fc..187c75b3 100644 --- a/pctiler/pctiler/endpoints/item.py +++ b/pctiler/pctiler/endpoints/item.py @@ -1,11 +1,10 @@ import logging -from typing import Annotated, Callable, Optional +from typing import Annotated, Optional from urllib.parse import quote_plus, urljoin import fastapi import pystac -import starlette -from fastapi import Body, Depends, HTTPException, Query, Request, Response +from fastapi import Body, Depends, Query, Request, Response from fastapi.templating import Jinja2Templates from geojson_pydantic.features import Feature from html_sanitizer.sanitizer import Sanitizer @@ -18,6 +17,7 @@ from pccommon.config import get_render_config from pctiler.colormaps import PCColorMapParams from pctiler.config import get_settings +from pctiler.endpoints.dependencies import get_endpoint_function from pctiler.reader import ItemSTACReader, ReaderParams try: @@ -158,17 +158,3 @@ def geojson_crop( # type: ignore env=env, ) return result - - -def get_endpoint_function( - router: fastapi.APIRouter, path: str, method: str -) -> Callable: - for route in router.routes: - match, _ = route.matches({"type": "http", "path": path, "method": method}) - if match == starlette.routing.Match.FULL: - # The abstract BaseRoute doesn't have a `.endpoint` attribute, - # but all of its subclasses do. - return route.endpoint # type: ignore [attr-defined] - - logger.warning(f"Could not find endpoint. method={method} path={path}") - raise HTTPException(detail="Internal system error", status_code=500) diff --git a/pctiler/pctiler/endpoints/pg_mosaic.py b/pctiler/pctiler/endpoints/pg_mosaic.py index 4e602adc..3dd77839 100644 --- a/pctiler/pctiler/endpoints/pg_mosaic.py +++ b/pctiler/pctiler/endpoints/pg_mosaic.py @@ -1,17 +1,22 @@ from dataclasses import dataclass, field -from typing import List, Optional +from typing import Annotated, List, Literal, Optional -from fastapi import FastAPI, Query, Request +from fastapi import APIRouter, Depends, FastAPI, Query, Request, Response from fastapi.responses import ORJSONResponse from psycopg_pool import ConnectionPool +from pydantic import Field from titiler.core import dependencies -from titiler.pgstac.dependencies import SearchIdParams +from titiler.core.dependencies import ColorFormulaParams +from titiler.core.factory import img_endpoint_params +from titiler.core.resources.enums import ImageType +from titiler.pgstac.dependencies import SearchIdParams, TmsTileParams from titiler.pgstac.factory import MosaicTilerFactory from pccommon.config import get_collection_config from pccommon.config.collections import MosaicInfo from pctiler.colormaps import PCColorMapParams from pctiler.config import get_settings +from pctiler.endpoints.dependencies import get_endpoint_function from pctiler.reader import PGSTACBackend, ReaderParams @@ -75,3 +80,90 @@ def mosaic_info( by_alias=True, exclude_unset=True ), ) + + +legacy_mosaic_router = APIRouter() + + +@legacy_mosaic_router.get("/tiles/{search_id}/{z}/{x}/{y}", **img_endpoint_params) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{z}/{x}/{y}.{format}", **img_endpoint_params +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{z}/{x}/{y}@{scale}x", **img_endpoint_params +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{z}/{x}/{y}@{scale}x.{format}", **img_endpoint_params +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{tileMatrixSetId}/{z}/{x}/{y}", **img_endpoint_params +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{tileMatrixSetId}/{z}/{x}/{y}.{format}", + **img_endpoint_params, +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x", + **img_endpoint_params, +) +@legacy_mosaic_router.get( + "/tiles/{search_id}/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", + **img_endpoint_params, +) +def tile_routes( # type: ignore + request: Request, + search_id=Depends(pgstac_mosaic_factory.path_dependency), + tile=Depends(TmsTileParams), + tileMatrixSetId: Annotated[ # type: ignore + Literal[tuple(pgstac_mosaic_factory.supported_tms.list())], + f"Identifier selecting one of the TileMatrixSetId supported (default: '{pgstac_mosaic_factory.default_tms}')", # noqa: E501,F722 + ] = pgstac_mosaic_factory.default_tms, + scale: Annotated[ # type: ignore + Optional[Annotated[int, Field(gt=0, le=4)]], + "Tile size scale. 1=256x256, 2=512x512...", # noqa: E501,F722 + ] = None, + format: Annotated[ + Optional[ImageType], + "Default will be automatically defined if the output image needs a mask (png) or not (jpeg).", # noqa: E501,F722 + ] = None, + layer_params=Depends(pgstac_mosaic_factory.layer_dependency), + dataset_params=Depends(pgstac_mosaic_factory.dataset_dependency), + pixel_selection=Depends(pgstac_mosaic_factory.pixel_selection_dependency), + tile_params=Depends(pgstac_mosaic_factory.tile_dependency), + post_process=Depends(pgstac_mosaic_factory.process_dependency), + rescale=Depends(pgstac_mosaic_factory.rescale_dependency), + color_formula=Depends(ColorFormulaParams), + colormap=Depends(pgstac_mosaic_factory.colormap_dependency), + render_params=Depends(pgstac_mosaic_factory.render_dependency), + pgstac_params=Depends(pgstac_mosaic_factory.pgstac_dependency), + backend_params=Depends(pgstac_mosaic_factory.backend_dependency), + reader_params=Depends(pgstac_mosaic_factory.reader_dependency), + env=Depends(pgstac_mosaic_factory.environment_dependency), +) -> Response: + """Create map tile.""" + endpoint = get_endpoint_function( + pgstac_mosaic_factory.router, + path="/tiles/{z}/{x}/{y}", + method=request.method, + ) + result = endpoint( + search_id=search_id, + tile=tile, + tileMatrixSetId=tileMatrixSetId, + scale=scale, + format=format, + tile_params=tile_params, + layer_params=layer_params, + dataset_params=dataset_params, + pixel_selection=pixel_selection, + post_process=post_process, + rescale=rescale, + color_formula=color_formula, + colormap=colormap, + render_params=render_params, + pgstac_params=pgstac_params, + backend_params=backend_params, + reader_params=reader_params, + env=env, + ) + return result diff --git a/pctiler/pctiler/main.py b/pctiler/pctiler/main.py index 682e3428..f6557303 100755 --- a/pctiler/pctiler/main.py +++ b/pctiler/pctiler/main.py @@ -71,6 +71,11 @@ async def lifespan(app: FastAPI) -> AsyncGenerator: prefix=settings.mosaic_endpoint_prefix + "/{search_id}", tags=["PgSTAC Mosaic endpoints"], ) +app.include_router( + pg_mosaic.legacy_mosaic_router, + prefix=settings.mosaic_endpoint_prefix, + tags=["PgSTAC Mosaic endpoints"], +) pg_mosaic.add_collection_mosaic_info_route( app, prefix=settings.mosaic_endpoint_prefix, diff --git a/pctiler/pctiler/middleware.py b/pctiler/pctiler/middleware.py index 7c5ccc4f..42255aa6 100644 --- a/pctiler/pctiler/middleware.py +++ b/pctiler/pctiler/middleware.py @@ -1,5 +1,4 @@ import json -from collections import OrderedDict from starlette.datastructures import MutableHeaders from starlette.types import ASGIApp, Message, Receive, Scope, Send @@ -23,11 +22,16 @@ async def send_with_searchid(message: Message) -> None: elif message_type == "http.response.body": # Rewrite id to searchid for backwards compatibility, keep key order body = json.loads(message["body"]) - ordered_body = OrderedDict() - ordered_body["searchid"] = body.get("id") - ordered_body.update(body) + body["searchid"] = body.get("id") - resp_body = json.dumps(ordered_body, ensure_ascii=False).encode("utf-8") + updated_links = [] + for link in body.get("links", []): + link["href"] = link["href"].replace("/{tileMatrixSetId}", "") + updated_links.append(link) + + body["links"] = updated_links + + resp_body = json.dumps(body, ensure_ascii=False).encode("utf-8") message["body"] = resp_body # Update the content-length header on the start message diff --git a/pctiler/tests/endpoints/test_pg_mosaic.py b/pctiler/tests/endpoints/test_pg_mosaic.py index e9b1e9c1..508bb5aa 100644 --- a/pctiler/tests/endpoints/test_pg_mosaic.py +++ b/pctiler/tests/endpoints/test_pg_mosaic.py @@ -1,20 +1,16 @@ +from typing import Any, Dict, Tuple + import pytest from httpx import AsyncClient from pccommon.config.collections import MosaicInfo +REGISTER_TYPE = Tuple[str, Dict[str, Any]] -@pytest.mark.asyncio -async def test_get(client: AsyncClient) -> None: - response = await client.get("/mosaic/info?collection=naip") - assert response.status_code == 200 - info_dict = response.json() - mosaic_info = MosaicInfo(**info_dict) - assert mosaic_info.default_location.zoom == 13 +@pytest.fixture +async def register_search(client: AsyncClient) -> REGISTER_TYPE: -@pytest.mark.asyncio -async def test_register(client: AsyncClient) -> None: cql = { "filter-lang": "cql2-json", "filter": { @@ -24,9 +20,107 @@ async def test_register(client: AsyncClient) -> None: } expected_content_hash = "8b989f86a149628eabfde894fb965982" response = await client.post("/mosaic/register", json=cql) - assert response.status_code == 200 resp = response.json() + return (expected_content_hash, resp) + + +@pytest.mark.asyncio +async def test_mosaic_info(client: AsyncClient) -> None: + response = await client.get("/mosaic/info?collection=naip") + assert response.status_code == 200 + info_dict = response.json() + mosaic_info = MosaicInfo(**info_dict) + assert mosaic_info.default_location.zoom == 13 + + +@pytest.mark.asyncio +async def test_register(client: AsyncClient, register_search: REGISTER_TYPE) -> None: + + expected_content_hash, register_resp = register_search + # Test that `searchid` which has been removed in titiler remains in pctiler, # and that the search hash remains consistent - assert resp["searchid"] == expected_content_hash + assert register_resp["searchid"] == expected_content_hash + # Test that the links have had the {tileMatrixSetId} template string removed + assert len(register_resp["links"]) == 2 + assert register_resp["links"][0]["href"].endswith( + f"/mosaic/{expected_content_hash}/tilejson.json" + ) + assert register_resp["links"][1]["href"].endswith( + f"/mosaic/{expected_content_hash}/WMTSCapabilities.xml" + ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + [ + # Legacy path routes + "mosaic/{searchId}/tiles/{z}/{x}/{y}", + "mosaic/{searchId}/tiles/{z}/{x}/{y}.{format}", + "mosaic/{searchId}/tiles/{z}/{x}/{y}@{scale}x", + "mosaic/{searchId}/tiles/{z}/{x}/{y}@{scale}x.{format}", + "mosaic/{searchId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}", + "mosaic/{searchId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}.{format}", + "mosaic/{searchId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x", + "mosaic/{searchId}/tiles/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", + # Current path routes + "mosaic/tiles/{searchId}/{z}/{x}/{y}", + "mosaic/tiles/{searchId}/{z}/{x}/{y}.{format}", + "mosaic/tiles/{searchId}/{z}/{x}/{y}@{scale}x", + "mosaic/tiles/{searchId}/{z}/{x}/{y}@{scale}x.{format}", + "mosaic/tiles/{searchId}/{tileMatrixSetId}/{z}/{x}/{y}", + "mosaic/tiles/{searchId}/{tileMatrixSetId}/{z}/{x}/{y}.{format}", + "mosaic/tiles/{searchId}/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x", + "mosaic/tiles/{searchId}/{tileMatrixSetId}/{z}/{x}/{y}@{scale}x.{format}", + ], +) +async def test_mosaic_tile_routes( + client: AsyncClient, register_search: REGISTER_TYPE, route: str +) -> None: + """ + For backwards compatibility, support both mosaic/tiles/{searchId} and + mosaic/{searchId}/tiles routes + """ + expected_content_hash, _ = register_search + + formatted_route = route.format( + searchId=expected_content_hash, + tileMatrixSetId="WebMercatorQuad", + z=16, + x=17218, + y=26838, + scale=2, + format="png", + ) + url = ( + f"/{formatted_route}?asset_bidx=image%7C1%2C2%2C3&assets=image&collection=naip" + ) + response = await client.get(url) + assert response.status_code == 200 + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "route", + [ + "mosaic/{searchId}/tilejson.json", + "mosaic/{searchId}/{tileMatrixSetId}/tilejson.json", + "mosaic/{searchId}/WMTSCapabilities.xml", + "mosaic/{searchId}/{tileMatrixSetId}/WMTSCapabilities.xml", + ], +) +async def test_tile_metadata_routes( + client: AsyncClient, register_search: REGISTER_TYPE, route: str +) -> None: + search_id, _ = register_search + + formatted_route = route.format( + searchId=search_id, tileMatrixSetId="WebMercatorQuad" + ) + url = ( + f"/{formatted_route}?asset_bidx=image%7C1%2C2%2C3&assets=image&collection=naip" + ) + response = await client.get(url) + assert response.status_code == 200