Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Vector Tiles: add a more generic way of building meta layer #71804

Merged
merged 1 commit into from
Apr 19, 2021
Merged

Vector Tiles: add a more generic way of building meta layer #71804

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions server/src/main/java/org/elasticsearch/common/util/Maps.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,10 @@
import org.elasticsearch.Assertions;

import java.util.Collection;
import java.util.HashMap;
import java.util.Map;
import java.util.Objects;
import java.util.TreeMap;
import java.util.stream.Collectors;
import java.util.stream.Stream;

Expand Down Expand Up @@ -125,4 +127,23 @@ public static <K, V> boolean deepEquals(Map<K, V> left, Map<K, V> right) {
.allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey())));
}

public static Map<String, Object> flatten(Map<String, Object> map, boolean ordered) {
return flatten(map, ordered, null);
}

@SuppressWarnings("unchecked")
private static Map<String, Object> flatten(Map<String, Object> map, boolean ordered, String parentPath) {
Map<String, Object> flatMap = ordered ? new TreeMap() : new HashMap<>();
String prefix = parentPath != null ? parentPath + "." : "";

for (Map.Entry<String, Object> entry : map.entrySet()) {
if (entry.getValue() instanceof Map) {
flatMap.putAll(flatten((Map<String, Object>) entry.getValue(), ordered, prefix + entry.getKey()));
} else {
flatMap.put(prefix + entry.getKey(), entry.getValue());
}
}

return flatMap;
}
}
77 changes: 60 additions & 17 deletions ...patial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,21 @@
import com.wdtinc.mapbox_vector_tile.encoding.MvtValue;
import org.elasticsearch.action.search.SearchRequestBuilder;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchResponseSections;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.geo.GeoBoundingBox;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeometryParser;
import org.elasticsearch.common.util.Maps;
import org.elasticsearch.common.xcontent.XContentHelper;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.geometry.Geometry;
import org.elasticsearch.geometry.Rectangle;
import org.elasticsearch.rest.RestRequest;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.Aggregations;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
Expand All @@ -35,8 +41,12 @@
import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation;
import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;
import org.elasticsearch.search.profile.SearchProfileShardResults;

import java.io.IOException;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import static org.elasticsearch.rest.RestRequest.Method.GET;

Expand Down Expand Up @@ -75,15 +85,43 @@ protected ResponseBuilder doParseRequest(RestRequest restRequest, Request reques
extent
);
final SearchHit[] hits = s.getHits().getHits();
final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null;
final InternalGeoBounds bounds = s.getAggregations() != null ? s.getAggregations().get(BOUNDS_FIELD) : null;
final Aggregations aggsWithoutGridAndBounds = s.getAggregations() == null
? null
: new Aggregations(
s.getAggregations()
.asList()
.stream()
.filter(a -> GRID_FIELD.equals(a.getName()) == false && BOUNDS_FIELD.equals(a.getName()) == false)
.collect(Collectors.toList())
);
SearchResponse meta = new SearchResponse(
new SearchResponseSections(
new SearchHits(SearchHits.EMPTY, s.getHits().getTotalHits(), s.getHits().getMaxScore()), // remove actual hits
aggsWithoutGridAndBounds,
s.getSuggest(),
s.isTimedOut(),
s.isTerminatedEarly(),
s.getProfileResults() == null ? null : new SearchProfileShardResults(s.getProfileResults()),
s.getNumReducePhases()
),
s.getScrollId(),
s.getTotalShards(),
s.getSuccessfulShards(),
s.getSkippedShards(),
s.getTook().millis(),
s.getShardFailures(),
s.getClusters()
);
if (hits.length > 0) {
tileBuilder.addLayers(getHitsLayer(s, request));
}
final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null;
// TODO: should be expose the total number of buckets on InternalGeoTileGrid?
if (grid != null && grid.getBuckets().size() > 0) {
tileBuilder.addLayers(getAggsLayer(s, request, geomBuilder));
tileBuilder.addLayers(getAggsLayer(grid, request, geomBuilder));
}
tileBuilder.addLayers(getMetaLayer(s, request, geomBuilder));
tileBuilder.addLayers(getMetaLayer(meta, bounds, request, geomBuilder));
tileBuilder.build().writeTo(b);
};
}
Expand Down Expand Up @@ -114,8 +152,8 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq
searchRequestBuilder.addAggregation(aBuilder);
}
if (request.getExactBounds()) {
final GeoBoundsAggregationBuilder boundsBuilder =
new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()).wrapLongitude(false);
final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField())
.wrapLongitude(false);
searchRequestBuilder.addAggregation(boundsBuilder);
}
return searchRequestBuilder;
Expand Down Expand Up @@ -148,11 +186,10 @@ private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Requ
return hitsLayerBuilder;
}

private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) {
private VectorTile.Tile.Layer.Builder getAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder) {
final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent());
final MvtLayerProps layerProps = new MvtLayerProps();
final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder();
final InternalGeoTileGrid grid = response.getAggregations().get(GRID_FIELD);
for (InternalGeoGridBucket<?> bucket : grid.getBuckets()) {
final long count = bucket.getDocCount();
featureBuilder.clear();
Expand All @@ -170,7 +207,7 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Requ
// Add aggregations results as key value pair
for (Aggregation aggregation : bucket.getAggregations()) {
final String type = aggregation.getType();
switch (type) {
switch (type) {
case MinAggregationBuilder.NAME:
case MaxAggregationBuilder.NAME:
case AvgAggregationBuilder.NAME:
Expand All @@ -190,11 +227,19 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Requ
return aggLayerBuilder;
}

private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) {
private VectorTile.Tile.Layer.Builder getMetaLayer(
SearchResponse response,
InternalGeoBounds bounds,
Request request,
VectorTileGeometryBuilder geomBuilder
) throws IOException {
Map<String, Object> responseMap = Maps.flatten(
XContentHelper.convertToMap(XContentHelper.toXContent(response, XContentType.CBOR, false), true, XContentType.CBOR).v2(),
true
);
final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent());
final MvtLayerProps layerProps = new MvtLayerProps();
final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder();
final InternalGeoBounds bounds = response.getAggregations() != null ? response.getAggregations().get(BOUNDS_FIELD) : null;
if (bounds != null && bounds.topLeft() != null) {
final GeoPoint topLeft = bounds.topLeft();
final GeoPoint bottomRight = bounds.bottomRight();
Expand All @@ -203,13 +248,11 @@ private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Requ
final Rectangle tile = request.getBoundingBox();
geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat());
}
addPropertyToFeature(featureBuilder, layerProps, "timed_out", response.isTimedOut());
addPropertyToFeature(featureBuilder, layerProps, "_shards.total", response.getTotalShards());
addPropertyToFeature(featureBuilder, layerProps, "_shards.successful", response.getSuccessfulShards());
addPropertyToFeature(featureBuilder, layerProps, "_shards.skipped", response.getSkippedShards());
addPropertyToFeature(featureBuilder, layerProps, "_shards.failed", response.getFailedShards());
addPropertyToFeature(featureBuilder, layerProps, "hits.total.value", response.getHits().getTotalHits().value);
addPropertyToFeature(featureBuilder, layerProps, "hits.total.relation", response.getHits().getTotalHits().relation.name());
for (Map.Entry<String, Object> entry : responseMap.entrySet()) {
if (entry.getValue() != null) {
addPropertyToFeature(featureBuilder, layerProps, entry.getKey(), entry.getValue());
}
}
metaLayerBuilder.addFeatures(featureBuilder);
addPropertiesToLayer(metaLayerBuilder, layerProps);
return metaLayerBuilder;
Expand Down
22 changes: 11 additions & 11 deletions ...lugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ public void testBasicGet() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);

}

Expand All @@ -141,7 +141,7 @@ public void testEmpty() throws Exception {
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY);
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(1));
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 10);
}

public void testGridPrecision() throws Exception {
Expand All @@ -152,7 +152,7 @@ public void testGridPrecision() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -170,7 +170,7 @@ public void testGridType() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -179,7 +179,7 @@ public void testGridType() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}
{
final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y);
Expand All @@ -195,7 +195,7 @@ public void testNoAggLayer() throws Exception {
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, HITS_LAYER, 4096, 33, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

public void testNoHitsLayer() throws Exception {
Expand All @@ -204,7 +204,7 @@ public void testNoHitsLayer() throws Exception {
final VectorTile.Tile tile = execute(mvtRequest);
assertThat(tile.getLayersCount(), Matchers.equalTo(2));
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 10);
}

public void testBasicQueryGet() throws Exception {
Expand All @@ -222,7 +222,7 @@ public void testBasicQueryGet() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 1, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

public void testBasicShape() throws Exception {
Expand All @@ -231,7 +231,7 @@ public void testBasicShape() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

public void testWithFields() throws Exception {
Expand All @@ -241,7 +241,7 @@ public void testWithFields() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 3);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

public void testMinAgg() throws Exception {
Expand All @@ -259,7 +259,7 @@ public void testMinAgg() throws Exception {
assertThat(tile.getLayersCount(), Matchers.equalTo(3));
assertLayer(tile, HITS_LAYER, 4096, 1, 1);
assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2);
assertLayer(tile, META_LAYER, 4096, 1, 7);
assertLayer(tile, META_LAYER, 4096, 1, 11);
}

private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) {
Expand Down