From 12817ed260fceabaf126706bd756394ea3dd1568 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Sun, 18 Apr 2021 16:19:43 -1000 Subject: [PATCH] Vector Tiles: add a more generic way of building meta layer Replaces a manual way of building the meta layer with a more generic way of collecting unprocessed elements of the search response --- .../org/elasticsearch/common/util/Maps.java | 21 +++++ .../vectortile/RestVectorTileAction.java | 77 +++++++++++++++---- .../xpack/spatial/VectorTileRestIT.java | 22 +++--- 3 files changed, 92 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/Maps.java b/server/src/main/java/org/elasticsearch/common/util/Maps.java index 30950e51756ae..167d3bd84297c 100644 --- a/server/src/main/java/org/elasticsearch/common/util/Maps.java +++ b/server/src/main/java/org/elasticsearch/common/util/Maps.java @@ -11,8 +11,10 @@ import org.elasticsearch.Assertions; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -125,4 +127,23 @@ public static boolean deepEquals(Map left, Map right) { .allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey()))); } + public static Map flatten(Map map, boolean ordered) { + return flatten(map, ordered, null); + } + + @SuppressWarnings("unchecked") + private static Map flatten(Map map, boolean ordered, String parentPath) { + Map flatMap = ordered ? new TreeMap() : new HashMap<>(); + String prefix = parentPath != null ? parentPath + "." : ""; + + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() instanceof Map) { + flatMap.putAll(flatten((Map) entry.getValue(), ordered, prefix + entry.getKey())); + } else { + flatMap.put(prefix + entry.getKey(), entry.getValue()); + } + } + + return flatMap; + } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java index 797342df69f78..8b0909cd5f5a0 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -12,15 +12,21 @@ import com.wdtinc.mapbox_vector_tile.encoding.MvtValue; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -35,8 +41,12 @@ import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.elasticsearch.search.profile.SearchProfileShardResults; +import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -75,15 +85,43 @@ protected ResponseBuilder doParseRequest(RestRequest restRequest, Request reques extent ); final SearchHit[] hits = s.getHits().getHits(); + final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null; + final InternalGeoBounds bounds = s.getAggregations() != null ? s.getAggregations().get(BOUNDS_FIELD) : null; + final Aggregations aggsWithoutGridAndBounds = s.getAggregations() == null + ? null + : new Aggregations( + s.getAggregations() + .asList() + .stream() + .filter(a -> GRID_FIELD.equals(a.getName()) == false && BOUNDS_FIELD.equals(a.getName()) == false) + .collect(Collectors.toList()) + ); + SearchResponse meta = new SearchResponse( + new SearchResponseSections( + new SearchHits(SearchHits.EMPTY, s.getHits().getTotalHits(), s.getHits().getMaxScore()), // remove actual hits + aggsWithoutGridAndBounds, + s.getSuggest(), + s.isTimedOut(), + s.isTerminatedEarly(), + s.getProfileResults() == null ? null : new SearchProfileShardResults(s.getProfileResults()), + s.getNumReducePhases() + ), + s.getScrollId(), + s.getTotalShards(), + s.getSuccessfulShards(), + s.getSkippedShards(), + s.getTook().millis(), + s.getShardFailures(), + s.getClusters() + ); if (hits.length > 0) { tileBuilder.addLayers(getHitsLayer(s, request)); } - final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null; // TODO: should be expose the total number of buckets on InternalGeoTileGrid? if (grid != null && grid.getBuckets().size() > 0) { - tileBuilder.addLayers(getAggsLayer(s, request, geomBuilder)); + tileBuilder.addLayers(getAggsLayer(grid, request, geomBuilder)); } - tileBuilder.addLayers(getMetaLayer(s, request, geomBuilder)); + tileBuilder.addLayers(getMetaLayer(meta, bounds, request, geomBuilder)); tileBuilder.build().writeTo(b); }; } @@ -114,8 +152,8 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq searchRequestBuilder.addAggregation(aBuilder); } if (request.getExactBounds()) { - final GeoBoundsAggregationBuilder boundsBuilder = - new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()).wrapLongitude(false); + final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) + .wrapLongitude(false); searchRequestBuilder.addAggregation(boundsBuilder); } return searchRequestBuilder; @@ -148,11 +186,10 @@ private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Requ return hitsLayerBuilder; } - private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) { + private VectorTile.Tile.Layer.Builder getAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder) { final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final InternalGeoTileGrid grid = response.getAggregations().get(GRID_FIELD); for (InternalGeoGridBucket bucket : grid.getBuckets()) { final long count = bucket.getDocCount(); featureBuilder.clear(); @@ -170,7 +207,7 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Requ // Add aggregations results as key value pair for (Aggregation aggregation : bucket.getAggregations()) { final String type = aggregation.getType(); - switch (type) { + switch (type) { case MinAggregationBuilder.NAME: case MaxAggregationBuilder.NAME: case AvgAggregationBuilder.NAME: @@ -190,11 +227,19 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Requ return aggLayerBuilder; } - private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) { + private VectorTile.Tile.Layer.Builder getMetaLayer( + SearchResponse response, + InternalGeoBounds bounds, + Request request, + VectorTileGeometryBuilder geomBuilder + ) throws IOException { + Map responseMap = Maps.flatten( + XContentHelper.convertToMap(XContentHelper.toXContent(response, XContentType.CBOR, false), true, XContentType.CBOR).v2(), + true + ); final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final InternalGeoBounds bounds = response.getAggregations() != null ? response.getAggregations().get(BOUNDS_FIELD) : null; if (bounds != null && bounds.topLeft() != null) { final GeoPoint topLeft = bounds.topLeft(); final GeoPoint bottomRight = bounds.bottomRight(); @@ -203,13 +248,11 @@ private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Requ final Rectangle tile = request.getBoundingBox(); geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); } - addPropertyToFeature(featureBuilder, layerProps, "timed_out", response.isTimedOut()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.total", response.getTotalShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.successful", response.getSuccessfulShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.skipped", response.getSkippedShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.failed", response.getFailedShards()); - addPropertyToFeature(featureBuilder, layerProps, "hits.total.value", response.getHits().getTotalHits().value); - addPropertyToFeature(featureBuilder, layerProps, "hits.total.relation", response.getHits().getTotalHits().relation.name()); + for (Map.Entry entry : responseMap.entrySet()) { + if (entry.getValue() != null) { + addPropertyToFeature(featureBuilder, layerProps, entry.getKey(), entry.getValue()); + } + } metaLayerBuilder.addFeatures(featureBuilder); addPropertiesToLayer(metaLayerBuilder, layerProps); return metaLayerBuilder; diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java index d5a776dd6d097..84e81a620cb92 100644 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java +++ b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java @@ -132,7 +132,7 @@ public void testBasicGet() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } @@ -141,7 +141,7 @@ public void testEmpty() throws Exception { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(1)); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 10); } public void testGridPrecision() throws Exception { @@ -152,7 +152,7 @@ public void testGridPrecision() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -170,7 +170,7 @@ public void testGridType() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -179,7 +179,7 @@ public void testGridType() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -195,7 +195,7 @@ public void testNoAggLayer() throws Exception { final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } public void testNoHitsLayer() throws Exception { @@ -204,7 +204,7 @@ public void testNoHitsLayer() throws Exception { final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 10); } public void testBasicQueryGet() throws Exception { @@ -222,7 +222,7 @@ public void testBasicQueryGet() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } public void testBasicShape() throws Exception { @@ -231,7 +231,7 @@ public void testBasicShape() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } public void testWithFields() throws Exception { @@ -241,7 +241,7 @@ public void testWithFields() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 3); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } public void testMinAgg() throws Exception { @@ -259,7 +259,7 @@ public void testMinAgg() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) {