Skip to content

Commit

Permalink
Vector Tiles: add a more generic way of building meta layer (#71804)
Browse files Browse the repository at this point in the history 
Replaces a manual way of building the meta layer with a more generic way
of collecting unprocessed elements of the search response.
  • Loading branch information
@imotov
imotov authored Apr 19, 2021
1 parent 14826ec commit 7b6f3da
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 28 deletions.
21 changes: 21 additions & 0 deletions server/src/main/java/org/elasticsearch/common/util/Maps.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@
import org.elasticsearch.Assertions;

import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -125,4 +127,23 @@ public static <K, V> boolean deepEquals(Map<K, V> left, Map<K, V> right) {
.allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey())));
}

public static Map<String, Object> flatten(Map<String, Object> map, boolean ordered) {
return flatten(map, ordered, null);
}

@SuppressWarnings("unchecked")
private static Map<String, Object> flatten(Map<String, Object> map, boolean ordered, String parentPath) {
Map<String, Object> flatMap = ordered ? new TreeMap() : new HashMap<>();
String prefix = parentPath != null ? parentPath + "." : "";

for (Map.Entry<String, Object> entry : map.entrySet()) {
if (entry.getValue() instanceof Map) {
flatMap.putAll(flatten((Map<String, Object>) entry.getValue(), ordered, prefix + entry.getKey()));
} else {
flatMap.put(prefix + entry.getKey(), entry.getValue());
}
}

return flatMap;
}
}
77 changes: 60 additions & 17 deletions ...patial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,21 @@
import com.wdtinc.mapbox_vector_tile.encoding.MvtValue;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchResponseSections;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.geo.GeoBoundingBox;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeometryParser;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.Rectangle;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
Expand All @@ -35,8 +41,12 @@
import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
import org.elasticsearch.search.profile.SearchProfileShardResults;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static org.elasticsearch.rest.RestRequest.Method.GET;

Expand Down Expand Up @@ -75,15 +85,43 @@ protected ResponseBuilder doParseRequest(RestRequest restRequest, Request reques
extent
);
final SearchHit[] hits = s.getHits().getHits();
final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null;
final InternalGeoBounds bounds = s.getAggregations() != null ? s.getAggregations().get(BOUNDS_FIELD) : null;
final Aggregations aggsWithoutGridAndBounds = s.getAggregations() == null
? null
: new Aggregations(
s.getAggregations()
.asList()
.stream()
.filter(a -> GRID_FIELD.equals(a.getName()) == false && BOUNDS_FIELD.equals(a.getName()) == false)
.collect(Collectors.toList())
);
SearchResponse meta = new SearchResponse(
new SearchResponseSections(
new SearchHits(SearchHits.EMPTY, s.getHits().getTotalHits(), s.getHits().getMaxScore()), // remove actual hits
aggsWithoutGridAndBounds,
s.getSuggest(),
s.isTimedOut(),
s.isTerminatedEarly(),
s.getProfileResults() == null ? null : new SearchProfileShardResults(s.getProfileResults()),
s.getNumReducePhases()
),
s.getScrollId(),
s.getTotalShards(),
s.getSuccessfulShards(),
s.getSkippedShards(),
s.getTook().millis(),
s.getShardFailures(),
s.getClusters()
);
if (hits.length > 0) {
tileBuilder.addLayers(getHitsLayer(s, request));
}
final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null;
// TODO: should be expose the total number of buckets on InternalGeoTileGrid?
if (grid != null && grid.getBuckets().size() > 0) {
tileBuilder.addLayers(getAggsLayer(s, request, geomBuilder));
tileBuilder.addLayers(getAggsLayer(grid, request, geomBuilder));
}
tileBuilder.addLayers(getMetaLayer(s, request, geomBuilder));
tileBuilder.addLayers(getMetaLayer(meta, bounds, request, geomBuilder));
tileBuilder.build().writeTo(b);
};
}
Expand Down Expand Up @@ -114,8 +152,8 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq
searchRequestBuilder.addAggregation(aBuilder);
}
if (request.getExactBounds()) {
final GeoBoundsAggregationBuilder boundsBuilder =
new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()).wrapLongitude(false);
final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField())
.wrapLongitude(false);
searchRequestBuilder.addAggregation(boundsBuilder);
}
return searchRequestBuilder;
Expand Down Expand Up @@ -148,11 +186,10 @@ private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Requ
return hitsLayerBuilder;
}

private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) {
private VectorTile.Tile.Layer.Builder getAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder) {
final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent());
final MvtLayerProps layerProps = new MvtLayerProps();
final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder();
final InternalGeoTileGrid grid = response.getAggregations().get(GRID_FIELD);
for (InternalGeoGridBucket<?> bucket : grid.getBuckets()) {
final long count = bucket.getDocCount();
featureBuilder.clear();
Expand All @@ -170,7 +207,7 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Requ
// Add aggregations results as key value pair
for (Aggregation aggregation : bucket.getAggregations()) {
final String type = aggregation.getType();
switch (type) {
switch (type) {
case MinAggregationBuilder.NAME:
case MaxAggregationBuilder.NAME:
case AvgAggregationBuilder.NAME:
Expand All @@ -190,11 +227,19 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Requ
return aggLayerBuilder;
}

private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) {
private VectorTile.Tile.Layer.Builder getMetaLayer(
SearchResponse response,
InternalGeoBounds bounds,
Request request,
VectorTileGeometryBuilder geomBuilder
) throws IOException {
Map<String, Object> responseMap = Maps.flatten(
XContentHelper.convertToMap(XContentHelper.toXContent(response, XContentType.CBOR, false), true, XContentType.CBOR).v2(),
true
);
final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent());
final MvtLayerProps layerProps = new MvtLayerProps();
final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder();
final InternalGeoBounds bounds = response.getAggregations() != null ? response.getAggregations().get(BOUNDS_FIELD) : null;
if (bounds != null && bounds.topLeft() != null) {
final GeoPoint topLeft = bounds.topLeft();
final GeoPoint bottomRight = bounds.bottomRight();
Expand All @@ -203,13 +248,11 @@ private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Requ
final Rectangle tile = request.getBoundingBox();
geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat());
}
addPropertyToFeature(featureBuilder, layerProps, "timed_out", response.isTimedOut());
addPropertyToFeature(featureBuilder, layerProps, "_shards.total", response.getTotalShards());
addPropertyToFeature(featureBuilder, layerProps, "_shards.successful", response.getSuccessfulShards());
addPropertyToFeature(featureBuilder, layerProps, "_shards.skipped", response.getSkippedShards());
addPropertyToFeature(featureBuilder, layerProps, "_shards.failed", response.getFailedShards());
addPropertyToFeature(featureBuilder, layerProps, "hits.total.value", response.getHits().getTotalHits().value);
addPropertyToFeature(featureBuilder, layerProps, "hits.total.relation", response.getHits().getTotalHits().relation.name());
for (Map.Entry<String, Object> entry : responseMap.entrySet()) {
if (entry.getValue() != null) {
addPropertyToFeature(featureBuilder, layerProps, entry.getKey(), entry.getValue());
}
}
metaLayerBuilder.addFeatures(featureBuilder);
addPropertiesToLayer(metaLayerBuilder, layerProps);
return metaLayerBuilder;
Expand Down
22 changes: 11 additions & 11 deletions ...lugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public void testBasicGet() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);

}

Expand All @@ -141,7 +141,7 @@ public void testEmpty() throws Exception {
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY);
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(1));
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 10);
}

public void testGridPrecision() throws Exception {
Expand All @@ -152,7 +152,7 @@ public void testGridPrecision() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -170,7 +170,7 @@ public void testGridType() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -179,7 +179,7 @@ public void testGridType() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -195,7 +195,7 @@ public void testNoAggLayer() throws Exception {
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

public void testNoHitsLayer() throws Exception {
Expand All @@ -204,7 +204,7 @@ public void testNoHitsLayer() throws Exception {
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 10);
}

public void testBasicQueryGet() throws Exception {
Expand All @@ -222,7 +222,7 @@ public void testBasicQueryGet() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

public void testBasicShape() throws Exception {
Expand All @@ -231,7 +231,7 @@ public void testBasicShape() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

public void testWithFields() throws Exception {
Expand All @@ -241,7 +241,7 @@ public void testWithFields() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 3);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

public void testMinAgg() throws Exception {
Expand All @@ -259,7 +259,7 @@ public void testMinAgg() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) {
Expand Down

0 comments on commit 7b6f3da

Please sign in to comment.