From b4e9dc8f3278e4cf6556be2532224219dc3a6a47 Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Apr 2021 09:57:36 +0200 Subject: [PATCH 1/3] simplify mvt end point --- x-pack/plugin/spatial/build.gradle | 11 - .../xpack/spatial/SpatialPlugin.java | 10 +- .../VectorTileGeoShapeAggregator.java | 8 +- .../AbstractVectorTileSearchAction.java | 149 +++++++++- .../spatial/vectortile/FeatureFactory.java | 10 +- .../RestAggregatedVectorTileAction.java | 209 ------------- .../vectortile/RestVectorTileAction.java | 232 ++++++++++++++- .../vectortile/VectorTileGeometryBuilder.java | 15 +- .../aggregations/InternalVectorTileTests.java | 9 +- .../xpack/spatial/VectorTileGridRestIT.java | 279 ------------------ .../xpack/spatial/VectorTileRestIT.java | 244 +++++++++++++-- 11 files changed, 604 insertions(+), 572 deletions(-) delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java delete mode 100644 x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index bef0731cf8a29..6072721b40c7d 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -36,17 +36,6 @@ testClusters.all { } tasks.named("thirdPartyAudit").configure { - ignoreViolations( - // uses internal java api: sun.misc.Unsafe - 'com.google.protobuf.UnsafeUtil', - 'com.google.protobuf.MessageSchema', - 'com.google.protobuf.UnsafeUtil$1', - 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', - 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', - 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', - 'com.google.protobuf.UnsafeUtil$MemoryAccessor' - ) - ignoreMissingClasses( 'org.slf4j.Logger', 'org.slf4j.LoggerFactory' diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 8b2fe7793fe84..0cde726f93fed 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -44,9 +44,6 @@ import org.elasticsearch.xpack.spatial.action.SpatialInfoTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialStatsTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialUsageTransportAction; -import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; -import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; -import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator; import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper; @@ -54,6 +51,8 @@ import org.elasticsearch.xpack.spatial.ingest.CircleProcessor; import org.elasticsearch.xpack.spatial.search.aggregations.GeoLineAggregationBuilder; import org.elasticsearch.xpack.spatial.search.aggregations.InternalGeoLine; +import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; +import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoHashGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoGridTiler; @@ -63,9 +62,9 @@ import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeTileGridAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; +import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; -import org.elasticsearch.xpack.spatial.vectortile.RestAggregatedVectorTileAction; import org.elasticsearch.xpack.spatial.vectortile.RestVectorTileAction; import java.util.Arrays; @@ -100,8 +99,7 @@ public List getRestHandlers(Settings settings, RestController restC IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster) { return Arrays.asList( - new RestVectorTileAction(), - new RestAggregatedVectorTileAction()); + new RestVectorTileAction()); } @Override diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java index 3a976fa885285..78a722861e342 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.spatial.search.aggregations; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.geometry.Point; @@ -68,6 +70,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc final FeatureFactory featureFactory = new FeatureFactory(z, x, y, POLYGON_EXTENT); final PointFactory pointFactory = new PointFactory(); final CustomFieldsVisitor visitor = new CustomFieldsVisitor(Set.of(), true); + IUserDataConverter ignoreData = new UserDataIgnoreConverter(); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -99,7 +102,7 @@ public void collect(int doc, long bucket) throws IOException { final Object lines = lookup.get(fieldName); if (lines != null) { addLineFeatures(visitor.id(), - featureFactory.getFeatures(parser.parseGeometry(lines))); + featureFactory.getFeatures(parser.parseGeometry(lines), ignoreData)); } } break; @@ -114,7 +117,8 @@ public void collect(int doc, long bucket) throws IOException { lookup.setSource(visitor.source()); final Object polygons = lookup.get(fieldName); if (polygons != null) { - addPolygonFeatures(visitor.id(), featureFactory.getFeatures(parser.parseGeometry(polygons))); + addPolygonFeatures(visitor.id(), + featureFactory.getFeatures(parser.parseGeometry(polygons), ignoreData)); } } break; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java index dcc39a00c502e..6dce260c1f2ae 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStream; @@ -27,13 +28,21 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestResponseListener; +import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.function.Supplier; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + /** * Base class for rest actions that performs a search and translates it into * a protobuf response @@ -45,6 +54,22 @@ public abstract class AbstractVectorTileSearchAction parser; @@ -56,12 +81,20 @@ protected interface ResponseBuilder { private final Supplier emptyRequestProvider; protected static class Request { - QueryBuilder queryBuilder; - String index; - String field; - int x; - int y; - int z; + private QueryBuilder queryBuilder; + private String index; + private String field; + private int x; + private int y; + private int z; + private Map runtimeMappings = emptyMap(); + private int gridPrecision = 8; + private GRID_TYPE gridType = GRID_TYPE.GRID; + private int size = 10000; + private int extent = 4096; + private AggregatorFactories.Builder aggBuilder; + private List fields = emptyList(); + private boolean exact_bounds; public String getIndex() { return index; @@ -103,11 +136,37 @@ public void setZ(int z) { this.z = z; } + public int getExtent() { + return extent; + } + + public void setExtent(int extent) { + // TODO: validation + this.extent = extent; + } + + public boolean getExactBounds() { + return exact_bounds; + } + + public void setExactBounds(boolean exact_bounds) { + this.exact_bounds = exact_bounds; + } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + public QueryBuilder getQueryBuilder() { return queryBuilder; } public void setQueryBuilder(QueryBuilder queryBuilder) { + // TODO: validation this.queryBuilder = queryBuilder; } @@ -125,17 +184,94 @@ public QueryBuilder getQuery() throws IOException { } return qBuilder; } + + public Map getRuntimeMappings() { + return runtimeMappings; + } + + public void setRuntimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + } + + public int getGridPrecision() { + return gridPrecision; + } + + public void setGridPrecision(int gridPrecision) { + if (gridPrecision < 0 || gridPrecision > 8) { + throw new IllegalArgumentException("Invalid grid precision, value should be between 0 and 8, got [" + gridPrecision + "]"); + } + this.gridPrecision = gridPrecision; + } + + public GRID_TYPE getGridType() { + return gridType; + } + + public void setGridType(String gridType) { + this.gridType = GRID_TYPE.fromString(gridType); + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + // TODO: validation + this.size = size; + } + + public AggregatorFactories.Builder getAggBuilder() { + return aggBuilder; + } + + public void setAggBuilder(AggregatorFactories.Builder aggBuilder) { + // TODO: validation + this.aggBuilder = aggBuilder; + } } protected AbstractVectorTileSearchAction(Supplier emptyRequestProvider) { this.emptyRequestProvider = emptyRequestProvider; parser = new ObjectParser<>(getName(), emptyRequestProvider); + parser.declareInt(Request::setSize, SearchSourceBuilder.SIZE_FIELD); + parser.declareField( + Request::setFields, + AbstractVectorTileSearchAction::parseFetchFields, + SearchSourceBuilder.FETCH_FIELDS_FIELD, + ObjectParser.ValueType.OBJECT_ARRAY + ); parser.declareField( Request::setQueryBuilder, (CheckedFunction) AbstractQueryBuilder::parseInnerQueryBuilder, SearchSourceBuilder.QUERY_FIELD, ObjectParser.ValueType.OBJECT ); + parser.declareField( + Request::setRuntimeMappings, + XContentParser::map, + SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + parser.declareField( + Request::setAggBuilder, + AggregatorFactories::parseAggregators, + SearchSourceBuilder.AGGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + // Specific for vector tiles + parser.declareInt(Request::setGridPrecision, GRID_PRECISION_FIELD); + parser.declareInt(Request::setExtent, EXTENT_FIELD); + parser.declareBoolean(Request::setExactBounds, EXACT_BOUNDS_FIELD); + parser.declareString(Request::setGridType, GRID_TYPE_FIELD); + } + + private static List parseFetchFields(XContentParser parser) throws IOException { + List fetchFields = new ArrayList<>(); + while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { + fetchFields.add(FieldAndFormat.fromXContent(parser)); + } + return fetchFields; } protected abstract ResponseBuilder doParseRequest(RestRequest restRequest, R request, SearchRequestBuilder searchRequestBuilder) @@ -159,7 +295,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient SearchRequestBuilder searchRequestBuilder = client.prepareSearch(Strings.splitStringByCommaToArray(request.getIndex())); searchRequestBuilder.setQuery(request.getQuery()); - searchRequestBuilder.setSize(0); ResponseBuilder responseBuilder = doParseRequest(restRequest, request, searchRequestBuilder); // TODO: how do we handle cancellations? diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java index 29077ca2d3258..c99cc238c2bf3 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java @@ -12,7 +12,6 @@ import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; import com.wdtinc.mapbox_vector_tile.adapt.jts.JtsAdapter; import com.wdtinc.mapbox_vector_tile.adapt.jts.TileGeomResult; -import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import com.wdtinc.mapbox_vector_tile.build.MvtLayerParams; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; import org.elasticsearch.geometry.Circle; @@ -37,7 +36,6 @@ public class FeatureFactory { private final IGeometryFilter acceptAllGeomFilter = geometry -> true; - private final IUserDataConverter ignoreUserData = new UserDataIgnoreConverter(); private final MvtLayerParams layerParams; private final GeometryFactory geomFactory = new GeometryFactory(); private final MvtLayerProps layerProps = new MvtLayerProps(); @@ -55,12 +53,16 @@ public FeatureFactory(int z, int x, int y, int extent) { this.layerParams = new MvtLayerParams(extent, extent); } - public List getFeatures(Geometry geometry) { + public List getFeatures(Geometry geometry, IUserDataConverter userData) { TileGeomResult tileGeom = JtsAdapter.createTileGeom(JtsAdapter.flatFeatureList(geometry.visit(builder)), tileEnvelope, clipEnvelope, geomFactory, layerParams, acceptAllGeomFilter); // MVT tile geometry to MVT features - return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, ignoreUserData); + return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, userData); + } + + public MvtLayerProps getLayerProps() { + return layerProps; } private static class JTSGeometryBuilder implements GeometryVisitor { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java deleted file mode 100644 index 7ea8e86d54f58..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.vectortile; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.GeoBoundingBox; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; -import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.Collections.emptyMap; -import static org.elasticsearch.rest.RestRequest.Method.GET; - -public class RestAggregatedVectorTileAction extends AbstractVectorTileSearchAction { - - private static final String TYPE_PARAM = "type"; - private static final String GRID_TYPE = "grid"; - - private static final String GRID_FIELD = "grid"; - private static final String BOUNDS_FIELD = "bounds"; - - private static final ParseField SCALING = new ParseField("scaling"); - - public RestAggregatedVectorTileAction() { - super(AggregatedRequest::new); - parser.declareField( - AggregatedRequest::setRuntimeMappings, - XContentParser::map, - SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, - ObjectParser.ValueType.OBJECT - ); - parser.declareField( - AggregatedRequest::setAggBuilder, - AggregatorFactories::parseAggregators, - SearchSourceBuilder.AGGS_FIELD, - ObjectParser.ValueType.OBJECT - ); - parser.declareInt(AggregatedRequest::setScaling, SCALING); - } - - protected static class AggregatedRequest extends AbstractVectorTileSearchAction.Request { - private Map runtimeMappings = emptyMap(); - private int scaling = 8; - private AggregatorFactories.Builder aggBuilder; - - public AggregatedRequest() {} - - public Map getRuntimeMappings() { - return runtimeMappings; - } - - public void setRuntimeMappings(Map runtimeMappings) { - this.runtimeMappings = runtimeMappings; - } - - public int getScaling() { - return scaling; - } - - public void setScaling(int scaling) { - this.scaling = scaling; - } - - public AggregatorFactories.Builder getAggBuilder() { - return aggBuilder; - } - - public void setAggBuilder(AggregatorFactories.Builder aggBuilder) { - this.aggBuilder = aggBuilder; - } - } - - @Override - public List routes() { - return List.of(new Route(GET, "{index}/_agg_mvt/{field}/{z}/{x}/{y}")); - } - - @Override - protected ResponseBuilder doParseRequest(RestRequest restRequest, AggregatedRequest request, SearchRequestBuilder searchRequestBuilder) - throws IOException { - final boolean isGrid = restRequest.hasParam(TYPE_PARAM) && GRID_TYPE.equals(restRequest.param(TYPE_PARAM)); - - searchBuilder(searchRequestBuilder, request); - final int extent = 1 << request.getScaling(); - - return (s, b) -> { - // TODO: of there is no hits, should we return an empty tile with no layers or - // a tile with empty layers? - final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); - final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder( - request.getZ(), - request.getX(), - request.getY(), - extent - ); - final InternalGeoTileGrid grid = s.getAggregations().get(GRID_FIELD); - tileBuilder.addLayers(getPointLayer(extent, isGrid, grid, geomBuilder)); - final InternalGeoBounds bounds = s.getAggregations().get(BOUNDS_FIELD); - tileBuilder.addLayers(getMetaLayer(extent, bounds, geomBuilder)); - tileBuilder.build().writeTo(b); - }; - } - - private VectorTile.Tile.Layer.Builder getPointLayer( - int extent, - boolean isGrid, - InternalGeoTileGrid t, - VectorTileGeometryBuilder geomBuilder - ) { - final VectorTile.Tile.Layer.Builder pointLayerBuilder = VectorTileUtils.createLayerBuilder("AGG", extent); - pointLayerBuilder.addKeys("count"); - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); - final HashMap values = new HashMap<>(); - - for (InternalGeoGridBucket bucket : t.getBuckets()) { - long count = bucket.getDocCount(); - if (count > 0) { - featureBuilder.clear(); - // create geometry commands - if (isGrid) { - Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); - geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); - } else { - GeoPoint point = (GeoPoint) bucket.getKey(); - geomBuilder.point(featureBuilder, point.lon(), point.lat()); - } - // Add count as key value pair - featureBuilder.addTags(0); - final int tagValue; - if (values.containsKey(count)) { - tagValue = values.get(count); - } else { - valueBuilder.clear(); - valueBuilder.setIntValue(count); - tagValue = values.size(); - pointLayerBuilder.addValues(valueBuilder); - values.put(count, tagValue); - } - featureBuilder.addTags(tagValue); - pointLayerBuilder.addFeatures(featureBuilder); - } - } - return pointLayerBuilder; - } - - private VectorTile.Tile.Layer.Builder getMetaLayer(int extent, InternalGeoBounds t, VectorTileGeometryBuilder geomBuilder) { - final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder("META", extent); - final GeoPoint topLeft = t.topLeft(); - final GeoPoint bottomRight = t.bottomRight(); - if (topLeft != null && bottomRight != null) { - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); - metaLayerBuilder.addFeatures(featureBuilder); - } - return metaLayerBuilder; - } - - private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchRequestBuilder, AggregatedRequest request) - throws IOException { - Rectangle rectangle = request.getBoundingBox(); - int extent = 1 << request.getScaling(); - GeoBoundingBox boundingBox = new GeoBoundingBox( - new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), - new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) - ); - GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) - .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getScaling())) - .setGeoBoundingBox(boundingBox) - .size(extent * extent); - if (request.getAggBuilder() != null) { - aBuilder.subAggregations(request.getAggBuilder()); - } - GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) - .wrapLongitude(false); - SearchRequestBuilder requestBuilder = searchRequestBuilder.addAggregation(aBuilder).addAggregation(boundsBuilder).setSize(0); - if (request.getRuntimeMappings() != null) { - requestBuilder.setRuntimeMappings(request.getRuntimeMappings()); - } - return requestBuilder; - } - - @Override - public String getName() { - return "vectortile_aggregation_action"; - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java index bd54609386568..61d74ccaa8b1e 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -6,17 +6,51 @@ */ package org.elasticsearch.xpack.spatial.vectortile; +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; +import com.wdtinc.mapbox_vector_tile.encoding.MvtValue; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; -import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; -import org.elasticsearch.xpack.spatial.vectortile.AbstractVectorTileSearchAction.Request; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; +import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -public class RestVectorTileAction extends AbstractVectorTileSearchAction { +public class RestVectorTileAction extends AbstractVectorTileSearchAction { + + private static final String META_LAYER = "meta"; + private static final String HITS_LAYER = "hits"; + private static final String AGGS_LAYER = "aggs"; + + private static final String GRID_FIELD = "grid"; + private static final String BOUNDS_FIELD = "bounds"; + + private static final String COUNT_TAG = "count"; + private static final String ID_TAG = "id"; public RestVectorTileAction() { super(Request::new); @@ -29,19 +63,191 @@ public List routes() { @Override protected ResponseBuilder doParseRequest(RestRequest restRequest, Request request, SearchRequestBuilder searchRequestBuilder) { - final VectorTileAggregationBuilder aBuilder = new VectorTileAggregationBuilder(request.getField()) - .field(request.getField()) - .z(request.getZ()) - .x(request.getX()) - .y(request.getY()); - searchRequestBuilder.addAggregation(aBuilder).setSize(0); + final int extent = request.getExtent(); + searchBuilder(searchRequestBuilder, request); return (s, b) -> { - InternalVectorTile t = s.getAggregations().get(request.getField()); - // TODO: Error processing - t.writeTileToStream(b); + // Even if there is no hits, we return a tile with the meta layer + final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); + final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder( + request.getZ(), + request.getX(), + request.getY(), + extent + ); + final SearchHit[] hits = s.getHits().getHits(); + if (hits.length > 0) { + tileBuilder.addLayers( + getHitsLayer(extent, request.getZ(), request.getX(), request.getY(), request.getField(), hits, request.getFields()) + ); + } + final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null; + // TODO: should be expose the total number of buckets on InternalGeoTileGrid? + if (grid != null && grid.getBuckets().size() > 0) { + tileBuilder.addLayers(getAggregationLayer(extent, request.getGridType(), grid, geomBuilder)); + } + tileBuilder.addLayers(getMetaLayer(extent, s, request.getBoundingBox(), geomBuilder)); + tileBuilder.build().writeTo(b); }; } + private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchRequestBuilder, Request request) { + searchRequestBuilder.setSize(request.getSize()); + searchRequestBuilder.setFetchSource(false); + // TODO: I wonder if we can leverage field and format so what we get in the result is already the mvt commands. + searchRequestBuilder.addFetchField(new FieldAndFormat(request.getField(), null)); + for (FieldAndFormat field : request.getFields()) { + searchRequestBuilder.addFetchField(field); + } + searchRequestBuilder.setRuntimeMappings(request.getRuntimeMappings()); + if (request.getGridPrecision() > 0) { + final Rectangle rectangle = request.getBoundingBox(); + final GeoBoundingBox boundingBox = new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + final int extent = 1 << request.getGridPrecision(); + final GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) + .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getGridPrecision())) + .setGeoBoundingBox(boundingBox) + .size(extent * extent); + if (request.getAggBuilder() != null) { + aBuilder.subAggregations(request.getAggBuilder()); + } + searchRequestBuilder.addAggregation(aBuilder); + } + if (request.getExactBounds()) { + final GeoBoundsAggregationBuilder boundsBuilder = + new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()).wrapLongitude(false); + searchRequestBuilder.addAggregation(boundsBuilder); + } + return searchRequestBuilder; + } + + private VectorTile.Tile.Layer.Builder getHitsLayer( + int extent, + int z, + int x, + int y, + String geoField, + SearchHit[] hits, + List fields + ) { + final FeatureFactory featureFactory = new FeatureFactory(z, x, y, extent); + final GeometryParser parser = new GeometryParser(true, false, false); + final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, extent); + for (SearchHit searchHit : hits) { + final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> { + // TODO: It would be great if we can add the centroid information for polygons. That information can be + // used to place labels inside those geometries + addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId()); + if (fields != null) { + for (FieldAndFormat field : fields) { + DocumentField documentField = searchHit.field(field.field); + if (documentField != null) { + addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue()); + } + } + } + }; + // TODO: See comment on field formats. + final Geometry geometry = parser.parseGeometry(searchHit.field(geoField).getValue()); + hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags)); + } + addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); + return hitsLayerBuilder; + } + + private VectorTile.Tile.Layer.Builder getAggregationLayer( + int extent, + GRID_TYPE grid_type, + InternalGeoTileGrid t, + VectorTileGeometryBuilder geomBuilder + ) { + final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, extent); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + for (InternalGeoGridBucket bucket : t.getBuckets()) { + final long count = bucket.getDocCount(); + featureBuilder.clear(); + // Add geometry + if (grid_type == GRID_TYPE.GRID) { + final Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); + geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + } else { + // TODO: it should be the centroid of the data + final GeoPoint point = (GeoPoint) bucket.getKey(); + geomBuilder.point(featureBuilder, point.lon(), point.lat()); + } + // Add count as key value pair + addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, count); + // Add aggregations results as key value pair + for (Aggregation aggregation : bucket.getAggregations()) { + final String type = aggregation.getType(); + switch (type) { + case MinAggregationBuilder.NAME: + case MaxAggregationBuilder.NAME: + case AvgAggregationBuilder.NAME: + case SumAggregationBuilder.NAME: + case CardinalityAggregationBuilder.NAME: + final NumericMetricsAggregation.SingleValue metric = (NumericMetricsAggregation.SingleValue) aggregation; + addPropertyToFeature(featureBuilder, layerProps, "aggs." + aggregation.getName(), metric.value()); + break; + default: + // top term and percentile should be supported + throw new IllegalArgumentException("Unknown feature type [" + type + "]"); + } + } + aggLayerBuilder.addFeatures(featureBuilder); + } + addPropertiesToLayer(aggLayerBuilder, layerProps); + return aggLayerBuilder; + } + + private VectorTile.Tile.Layer.Builder getMetaLayer( + int extent, + SearchResponse searchResponse, + Rectangle tile, + VectorTileGeometryBuilder geomBuilder + ) { + final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, extent); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + final InternalGeoBounds bounds + = searchResponse.getAggregations() != null ? searchResponse.getAggregations().get(BOUNDS_FIELD) : null; + if (bounds != null && bounds.topLeft() != null) { + final GeoPoint topLeft = bounds.topLeft(); + final GeoPoint bottomRight = bounds.bottomRight(); + geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); + } else { + geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); + } + addPropertyToFeature(featureBuilder, layerProps, "timed_out", searchResponse.isTimedOut()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.total", searchResponse.getTotalShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.successful", searchResponse.getSuccessfulShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.skipped", searchResponse.getSkippedShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.failed", searchResponse.getFailedShards()); + addPropertyToFeature(featureBuilder, layerProps, "hits.total.value", searchResponse.getHits().getTotalHits().value); + addPropertyToFeature(featureBuilder, layerProps, "hits.total.relation", searchResponse.getHits().getTotalHits().relation.name()); + metaLayerBuilder.addFeatures(featureBuilder); + addPropertiesToLayer(metaLayerBuilder, layerProps); + return metaLayerBuilder; + } + + private void addPropertyToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, String key, Object value) { + feature.addTags(layerProps.addKey(key)); + feature.addTags(layerProps.addValue(value)); + } + + private void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, MvtLayerProps layerProps) { + // Add keys + layer.addAllKeys(layerProps.getKeys()); + // Add values + final Iterable values = layerProps.getVals(); + for (Object value : values) { + layer.addValues(MvtValue.toValue(value)); + } + } + @Override public String getName() { return "vectortile_action"; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java index ea1fcc5a7b01f..5d31a7480da5a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java @@ -26,14 +26,6 @@ class VectorTileGeometryBuilder { pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) extent); } - public int lat(double lat) { - return (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; - } - - public int lon(double lon) { - return (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); - } - public void point(VectorTile.Tile.Feature.Builder featureBuilder, double lon, double lat) { featureBuilder.setType(VectorTile.Tile.GeomType.POINT); featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.MoveTo, 1)); @@ -64,4 +56,11 @@ public void box(VectorTile.Tile.Feature.Builder featureBuilder, double minLon, d featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.ClosePath, 1)); } + private int lat(double lat) { + return (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; + } + + private int lon(double lon) { + return (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java index 18f297402e1dc..3957dd3b8472f 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.spatial.search.aggregations; import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -40,11 +42,12 @@ static VectorTile.Tile.Layer randomPolygonLayer(int shapes) { layerBuilder.setName(AbstractVectorTileAggregator.POLYGON_LAYER); layerBuilder.setExtent(AbstractVectorTileAggregator.POLYGON_EXTENT); final FeatureFactory factory = new FeatureFactory(0, 0, 0, AbstractVectorTileAggregator.POLYGON_EXTENT); + final IUserDataConverter ignoreData = new UserDataIgnoreConverter(); for (int i =0; i < shapes; i++) { int count = layerBuilder.getFeaturesCount(); while(true) { Geometry geometry = GeometryTestUtils.randomPolygon(false); - List features = factory.getFeatures(geometry); + List features = factory.getFeatures(geometry, ignoreData); for (VectorTile.Tile.Feature feature : features) { layerBuilder.addFeatures(feature); } @@ -54,7 +57,6 @@ static VectorTile.Tile.Layer randomPolygonLayer(int shapes) { } } return layerBuilder.build(); - } static VectorTile.Tile.Layer randomLineLayer(int shapes) { @@ -63,11 +65,12 @@ static VectorTile.Tile.Layer randomLineLayer(int shapes) { layerBuilder.setName(AbstractVectorTileAggregator.POLYGON_LAYER); layerBuilder.setExtent(AbstractVectorTileAggregator.POLYGON_EXTENT); final FeatureFactory factory = new FeatureFactory(0, 0, 0, AbstractVectorTileAggregator.POLYGON_EXTENT); + final IUserDataConverter ignoreData = new UserDataIgnoreConverter(); for (int i =0; i < shapes; i++) { int count = layerBuilder.getFeaturesCount(); while(true) { Geometry geometry = GeometryTestUtils.randomLine(false); - List features = factory.getFeatures(geometry); + List features = factory.getFeatures(geometry, ignoreData); for (VectorTile.Tile.Feature feature : features) { layerBuilder.addFeatures(feature); } diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java deleted file mode 100644 index d096b7e60ae1c..0000000000000 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.hamcrest.Matchers; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.io.InputStream; - -public class VectorTileGridRestIT extends ESRestTestCase { - - private static String INDEX_POINTS = "index-points"; - private static String INDEX_SHAPES = "index-shapes"; - - private int x, y, z; - - @Before - public void indexDocuments() throws IOException { - - z = randomIntBetween(1, GeoTileUtils.MAX_ZOOM - 10); - x = randomIntBetween(0, (1 << z) - 1); - y = randomIntBetween(0, (1 << z) - 1); - indexPoints(); - indexShapes(); - } - - private void indexPoints() throws IOException { - - final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS); - Response response = client().performRequest(createRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - - final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS + "/_mapping"); - mappingRequest.setJsonEntity("{\n" + - " \"properties\": {\n" + - " \"location\": {\n" + - " \"type\": \"geo_point\"\n" + - " },\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + - "}"); - response = client().performRequest(mappingRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); - double x = (r.getMaxX() + r.getMinX()) / 2; - double y = (r.getMaxY() + r.getMinY()) / 2; - for (int i = 0; i < 30; i+=10) { - for (int j = 0; j <= i; j++) { - final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_doc"); - putRequest.setJsonEntity("{\n" + - " \"location\": \"POINT(" + x + " " + y + ")\", \"name\": \"point" + i + "\"" + - ", \"value1\": " + i + ", \"value2\": " + (i + 1) + "\n" + - "}"); - response = client().performRequest(putRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); - } - } - - final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_refresh"); - response = client().performRequest(flushRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - } - - private void indexShapes() throws IOException { - - final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES); - Response response = client().performRequest(createRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - - final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES + "/_mapping"); - mappingRequest.setJsonEntity("{\n" + - " \"properties\": {\n" + - " \"location\": {\n" + - " \"type\": \"geo_shape\"\n" + - " },\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + - "}"); - response = client().performRequest(mappingRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - - Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); - - final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_doc"); - putRequest.setJsonEntity("{\n" + - " \"location\": \"BBOX (" + r.getMinLon() + ", " + r.getMaxLon() + "," + r.getMaxLat() + "," + r.getMinLat() + ")\"" + - ", \"name\": \"rectangle\"" + - ", \"value1\": " + 1 + ", \"value2\": " + 2 + "\n" + - "}"); - response = client().performRequest(putRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); - - - final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_refresh"); - response = client().performRequest(flushRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - } - - @After - public void deleteData() throws IOException { - final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_POINTS); - Response response = client().performRequest(deleteRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - } - - public void testBasicGet() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } - - public void testEmpty() throws Exception { - final int newY = (1 << z) - 1 == y ? y - 1 : y + 1; - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + newY); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(0)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(0)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(0)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } - - public void testBasicScaling() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"scaling\": 7 }"); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(128)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(128)); - } - } - - private VectorTile.Tile.Layer getLayer(VectorTile.Tile tile, String layerName) { - for (int i = 0; i < tile.getLayersCount(); i++) { - VectorTile.Tile.Layer layer = tile.getLayers(i); - if (layerName.equals(layer.getName())) { - return layer; - } - } - fail("Could not find layer " + layerName); - return null; - } - - @AwaitsFix(bugUrl = "doesn't work yet") - public void testBasicQueryGet() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\n" + - " \"query\": {\n" + - " \"term\": {\n" + - " \"name\": {\n" + - " \"value\": \"point2\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"); - Response response = client().performRequest(mvtRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - InputStream inputStream = response.getEntity().getContent(); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } - - public void testBasicShape() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_agg_mvt/location/"+ z + "/" + x + "/" + y); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(256 * 256)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } - - public void testMinAgg() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_agg_mvt/location/"+ z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\n" + - " \"aggs\": {\n" + - " \"minVal\": {\n" + - " \"min\": {\n" + - " \"field\": \"value1\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(256 * 256)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } -} diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java index 6e4af5d93f3e5..d5a776dd6d097 100644 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java +++ b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java @@ -15,6 +15,9 @@ import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.hamcrest.Matchers; import org.junit.After; @@ -23,18 +26,34 @@ import java.io.IOException; import java.io.InputStream; +/** + * Rest test for _mvt end point. The test only check that the structure of the vector tiles is sound in + * respect to the number of layers returned and the number of features abd tags in each layer. + */ public class VectorTileRestIT extends ESRestTestCase { - private static String INDEX_NAME = "my-index"; + private static final String INDEX_POINTS = "index-points"; + private static final String INDEX_SHAPES = "index-shapes"; + private static final String META_LAYER = "meta"; + private static final String HITS_LAYER = "hits"; + private static final String AGGS_LAYER = "aggs"; + + private int x, y, z; @Before public void indexDocuments() throws IOException { + z = randomIntBetween(1, GeoTileUtils.MAX_ZOOM - 10); + x = randomIntBetween(0, (1 << z) - 1); + y = randomIntBetween(0, (1 << z) - 1); + indexPoints(); + indexShapes(); + } - final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_NAME); + private void indexPoints() throws IOException { + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS); Response response = client().performRequest(createRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - - final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_NAME + "/_mapping"); + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS + "/_mapping"); mappingRequest.setJsonEntity("{\n" + " \"properties\": {\n" + " \"location\": {\n" + @@ -47,59 +66,224 @@ public void indexDocuments() throws IOException { "}"); response = client().performRequest(mappingRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + double x = (r.getMaxX() + r.getMinX()) / 2; + double y = (r.getMaxY() + r.getMinY()) / 2; + for (int i = 0; i < 30; i += 10) { + for (int j = 0; j <= i; j++) { + final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_doc"); + putRequest.setJsonEntity("{\n" + + " \"location\": \"POINT(" + x + " " + y + ")\", \"name\": \"point" + i + "\"" + + ", \"value1\": " + i + ", \"value2\": " + (i + 1) + "\n" + + "}"); + response = client().performRequest(putRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + } + } - final Request putRequest1 = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_doc"); - putRequest1.setJsonEntity("{\n" + - " \"location\": \"POINT(0 0)\", \"name\": \"point1\"\n" + - "}"); - - response = client().performRequest(putRequest1); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_refresh"); + response = client().performRequest(flushRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } - final Request putRequest2 = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_doc"); - putRequest2.setJsonEntity("{\n" + - " \"location\": \"POINT(1 1)\", \"name\": \"point2\"\n" + + private void indexShapes() throws IOException { + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES); + Response response = client().performRequest(createRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES + "/_mapping"); + mappingRequest.setJsonEntity("{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_shape\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + "}"); + response = client().performRequest(mappingRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - response = client().performRequest(putRequest2); + final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_doc"); + putRequest.setJsonEntity("{\n" + + " \"location\": \"BBOX (" + r.getMinLon() + ", " + r.getMaxLon() + "," + r.getMaxLat() + "," + r.getMinLat() + ")\"" + + ", \"name\": \"rectangle\"" + + ", \"value1\": " + 1 + ", \"value2\": " + 2 + "\n" + + "}"); + response = client().performRequest(putRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); - final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_refresh"); + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_refresh"); response = client().performRequest(flushRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); } @After public void deleteData() throws IOException { - final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_NAME); - Response response = client().performRequest(deleteRequest); + final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_POINTS); + final Response response = client().performRequest(deleteRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); } public void testBasicGet() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_NAME + "/_mvt/location/0/0/0"); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile.Builder builder = VectorTile.Tile.newBuilder().mergeFrom(inputStream); - assertThat(builder.getLayers(0).getFeaturesCount(), Matchers.equalTo(2)); + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + + } + + public void testEmpty() throws Exception { + final int newY = (1 << z) - 1 == y ? y - 1 : y + 1; + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(1)); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testGridPrecision() throws Exception { + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_precision\": 7 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_precision\": 9 }"); + final ResponseException ex = expectThrows(ResponseException.class, () -> execute(mvtRequest)); + assertThat(ex.getResponse().getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_BAD_REQUEST)); + } + } + + public void testGridType() throws Exception { + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_type\": \"point\" }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_type\": \"grid\" }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_type\": \"invalid_type\" }"); + final ResponseException ex = expectThrows(ResponseException.class, () -> execute(mvtRequest)); + assertThat(ex.getResponse().getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_BAD_REQUEST)); + } + } + + public void testNoAggLayer() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_precision\": 0 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testNoHitsLayer() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\": 0 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); } public void testBasicQueryGet() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_NAME + "/_mvt/location/0/0/0"); + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); mvtRequest.setJsonEntity("{\n" + " \"query\": {\n" + " \"term\": {\n" + " \"name\": {\n" + - " \"value\": \"point2\"\n" + + " \"value\": \"point0\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testBasicShape() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testWithFields() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"fields\": [\"name\", \"value1\"] }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 3); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testMinAgg() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\n" + + " \"aggs\": {\n" + + " \"minVal\": {\n" + + " \"min\": {\n" + + " \"field\": \"value1\"\n" + " }\n" + " }\n" + " }\n" + "}"); - Response response = client().performRequest(mvtRequest); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) { + final VectorTile.Tile.Layer layer = getLayer(tile, name); + assertThat(layer.getExtent(), Matchers.equalTo(extent)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(numFeatures)); + assertThat(layer.getKeysCount(), Matchers.equalTo(numTags)); + } + + private VectorTile.Tile execute(Request mvtRequest) throws IOException { + final Response response = client().performRequest(mvtRequest); + final InputStream inputStream = response.getEntity().getContent(); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - InputStream inputStream = response.getEntity().getContent(); - VectorTile.Tile.Builder builder = VectorTile.Tile.newBuilder().mergeFrom(inputStream); - assertThat(builder.getLayers(0).getFeaturesCount(), Matchers.equalTo(1)); + return VectorTile.Tile.parseFrom(inputStream); + } + + private VectorTile.Tile.Layer getLayer(VectorTile.Tile tile, String layerName) { + for (int i = 0; i < tile.getLayersCount(); i++) { + final VectorTile.Tile.Layer layer = tile.getLayers(i); + if (layerName.equals(layer.getName())) { + return layer; + } + } + fail("Could not find layer " + layerName); + return null; } } From 08454ce309d4a7a6600de327ceb6cae7de8a3f34 Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Apr 2021 10:32:43 +0200 Subject: [PATCH 2/3] restore ignoreViolations --- x-pack/plugin/spatial/build.gradle | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 6072721b40c7d..bef0731cf8a29 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -36,6 +36,17 @@ testClusters.all { } tasks.named("thirdPartyAudit").configure { + ignoreViolations( + // uses internal java api: sun.misc.Unsafe + 'com.google.protobuf.UnsafeUtil', + 'com.google.protobuf.MessageSchema', + 'com.google.protobuf.UnsafeUtil$1', + 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', + 'com.google.protobuf.UnsafeUtil$MemoryAccessor' + ) + ignoreMissingClasses( 'org.slf4j.Logger', 'org.slf4j.LoggerFactory' From 11566815fe53a6782c28a15db428d243419a9f83 Mon Sep 17 00:00:00 2001 From: iverase Date: Mon, 12 Apr 2021 12:49:36 +0200 Subject: [PATCH 3/3] simplification --- .../vectortile/RestVectorTileAction.java | 68 +++++++------------ 1 file changed, 25 insertions(+), 43 deletions(-) diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java index 61d74ccaa8b1e..797342df69f78 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -76,16 +76,14 @@ protected ResponseBuilder doParseRequest(RestRequest restRequest, Request reques ); final SearchHit[] hits = s.getHits().getHits(); if (hits.length > 0) { - tileBuilder.addLayers( - getHitsLayer(extent, request.getZ(), request.getX(), request.getY(), request.getField(), hits, request.getFields()) - ); + tileBuilder.addLayers(getHitsLayer(s, request)); } final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null; // TODO: should be expose the total number of buckets on InternalGeoTileGrid? if (grid != null && grid.getBuckets().size() > 0) { - tileBuilder.addLayers(getAggregationLayer(extent, request.getGridType(), grid, geomBuilder)); + tileBuilder.addLayers(getAggsLayer(s, request, geomBuilder)); } - tileBuilder.addLayers(getMetaLayer(extent, s, request.getBoundingBox(), geomBuilder)); + tileBuilder.addLayers(getMetaLayer(s, request, geomBuilder)); tileBuilder.build().writeTo(b); }; } @@ -123,19 +121,12 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq return searchRequestBuilder; } - private VectorTile.Tile.Layer.Builder getHitsLayer( - int extent, - int z, - int x, - int y, - String geoField, - SearchHit[] hits, - List fields - ) { - final FeatureFactory featureFactory = new FeatureFactory(z, x, y, extent); + private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Request request) { + final FeatureFactory featureFactory = new FeatureFactory(request.getZ(), request.getX(), request.getY(), request.getExtent()); final GeometryParser parser = new GeometryParser(true, false, false); - final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, extent); - for (SearchHit searchHit : hits) { + final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, request.getExtent()); + final List fields = request.getFields(); + for (SearchHit searchHit : response.getHits()) { final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> { // TODO: It would be great if we can add the centroid information for polygons. That information can be // used to place labels inside those geometries @@ -150,27 +141,23 @@ private VectorTile.Tile.Layer.Builder getHitsLayer( } }; // TODO: See comment on field formats. - final Geometry geometry = parser.parseGeometry(searchHit.field(geoField).getValue()); + final Geometry geometry = parser.parseGeometry(searchHit.field(request.getField()).getValue()); hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags)); } addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); return hitsLayerBuilder; } - private VectorTile.Tile.Layer.Builder getAggregationLayer( - int extent, - GRID_TYPE grid_type, - InternalGeoTileGrid t, - VectorTileGeometryBuilder geomBuilder - ) { - final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, extent); + private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) { + final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - for (InternalGeoGridBucket bucket : t.getBuckets()) { + final InternalGeoTileGrid grid = response.getAggregations().get(GRID_FIELD); + for (InternalGeoGridBucket bucket : grid.getBuckets()) { final long count = bucket.getDocCount(); featureBuilder.clear(); // Add geometry - if (grid_type == GRID_TYPE.GRID) { + if (request.getGridType() == GRID_TYPE.GRID) { final Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); } else { @@ -203,31 +190,26 @@ private VectorTile.Tile.Layer.Builder getAggregationLayer( return aggLayerBuilder; } - private VectorTile.Tile.Layer.Builder getMetaLayer( - int extent, - SearchResponse searchResponse, - Rectangle tile, - VectorTileGeometryBuilder geomBuilder - ) { - final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, extent); + private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) { + final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final InternalGeoBounds bounds - = searchResponse.getAggregations() != null ? searchResponse.getAggregations().get(BOUNDS_FIELD) : null; + final InternalGeoBounds bounds = response.getAggregations() != null ? response.getAggregations().get(BOUNDS_FIELD) : null; if (bounds != null && bounds.topLeft() != null) { final GeoPoint topLeft = bounds.topLeft(); final GeoPoint bottomRight = bounds.bottomRight(); geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); } else { + final Rectangle tile = request.getBoundingBox(); geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); } - addPropertyToFeature(featureBuilder, layerProps, "timed_out", searchResponse.isTimedOut()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.total", searchResponse.getTotalShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.successful", searchResponse.getSuccessfulShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.skipped", searchResponse.getSkippedShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.failed", searchResponse.getFailedShards()); - addPropertyToFeature(featureBuilder, layerProps, "hits.total.value", searchResponse.getHits().getTotalHits().value); - addPropertyToFeature(featureBuilder, layerProps, "hits.total.relation", searchResponse.getHits().getTotalHits().relation.name()); + addPropertyToFeature(featureBuilder, layerProps, "timed_out", response.isTimedOut()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.total", response.getTotalShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.successful", response.getSuccessfulShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.skipped", response.getSkippedShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.failed", response.getFailedShards()); + addPropertyToFeature(featureBuilder, layerProps, "hits.total.value", response.getHits().getTotalHits().value); + addPropertyToFeature(featureBuilder, layerProps, "hits.total.relation", response.getHits().getTotalHits().relation.name()); metaLayerBuilder.addFeatures(featureBuilder); addPropertiesToLayer(metaLayerBuilder, layerProps); return metaLayerBuilder;