Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

simplify mvt end point #71548

Merged
merged 3 commits into from
Apr 13, 2021
Merged

simplify mvt end point #71548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b4e9dc8
simplify mvt end point
iverase Apr 12, 2021
08454ce
restore ignoreViolations
iverase Apr 12, 2021
1156681
simplification
iverase Apr 12, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,16 +44,15 @@
import org.elasticsearch.xpack.spatial.action.SpatialInfoTransportAction;
import org.elasticsearch.xpack.spatial.action.SpatialStatsTransportAction;
import org.elasticsearch.xpack.spatial.action.SpatialUsageTransportAction;
import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile;
import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder;
import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator;
import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper;
import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper;
import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper;
import org.elasticsearch.xpack.spatial.index.query.ShapeQueryBuilder;
import org.elasticsearch.xpack.spatial.ingest.CircleProcessor;
import org.elasticsearch.xpack.spatial.search.aggregations.GeoLineAggregationBuilder;
import org.elasticsearch.xpack.spatial.search.aggregations.InternalGeoLine;
import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile;
import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder;
import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoHashGridTiler;
import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler;
import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoGridTiler;
Expand All @@ -63,9 +62,9 @@
import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeTileGridAggregator;
import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoTileGridTiler;
import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator;
import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator;
import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource;
import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType;
import org.elasticsearch.xpack.spatial.vectortile.RestAggregatedVectorTileAction;
import org.elasticsearch.xpack.spatial.vectortile.RestVectorTileAction;

import java.util.Arrays;
Expand Down Expand Up @@ -100,8 +99,7 @@ public List<RestHandler> getRestHandlers(Settings settings, RestController restC
IndexNameExpressionResolver indexNameExpressionResolver,
Supplier<DiscoveryNodes> nodesInCluster) {
return Arrays.asList(
new RestVectorTileAction(),
new RestAggregatedVectorTileAction());
new RestVectorTileAction());
}

@Override
Expand Down
8 changes: 6 additions & 2 deletions ...ava/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

package org.elasticsearch.xpack.spatial.search.aggregations;

import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter;
import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter;
import org.apache.lucene.index.LeafReaderContext;
import org.elasticsearch.common.geo.GeometryParser;
import org.elasticsearch.geometry.Point;
Expand Down Expand Up @@ -68,6 +70,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc
final FeatureFactory featureFactory = new FeatureFactory(z, x, y, POLYGON_EXTENT);
final PointFactory pointFactory = new PointFactory();
final CustomFieldsVisitor visitor = new CustomFieldsVisitor(Set.of(), true);
IUserDataConverter ignoreData = new UserDataIgnoreConverter();
return new LeafBucketCollectorBase(sub, values) {
@Override
public void collect(int doc, long bucket) throws IOException {
Expand Down Expand Up @@ -99,7 +102,7 @@ public void collect(int doc, long bucket) throws IOException {
final Object lines = lookup.get(fieldName);
if (lines != null) {
addLineFeatures(visitor.id(),
featureFactory.getFeatures(parser.parseGeometry(lines)));
featureFactory.getFeatures(parser.parseGeometry(lines), ignoreData));
}
}
break;
Expand All @@ -114,7 +117,8 @@ public void collect(int doc, long bucket) throws IOException {
lookup.setSource(visitor.source());
final Object polygons = lookup.get(fieldName);
if (polygons != null) {
addPolygonFeatures(visitor.id(), featureFactory.getFeatures(parser.parseGeometry(polygons)));
addPolygonFeatures(visitor.id(),
featureFactory.getFeatures(parser.parseGeometry(polygons), ignoreData));
}
}
break;
Expand Down
149 changes: 142 additions & 7 deletions .../main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.node.NodeClient;
import org.elasticsearch.common.CheckedFunction;
import org.elasticsearch.common.ParseField;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.io.Streams;
import org.elasticsearch.common.io.stream.BytesStream;
Expand All @@ -27,13 +28,21 @@
import org.elasticsearch.rest.RestResponse;
import org.elasticsearch.rest.RestStatus;
import org.elasticsearch.rest.action.RestResponseListener;
import org.elasticsearch.search.aggregations.AggregatorFactories;
import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;

import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Supplier;

import static java.util.Collections.emptyList;
import static java.util.Collections.emptyMap;

/**
* Base class for rest actions that performs a search and translates it into
* a protobuf response
Expand All @@ -45,6 +54,22 @@ public abstract class AbstractVectorTileSearchAction<R extends AbstractVectorTil
private static final String Z_PARAM = "z";
private static final String X_PARAM = "x";
private static final String Y_PARAM = "y";
private static final ParseField GRID_PRECISION_FIELD = new ParseField("grid_precision");
private static final ParseField GRID_TYPE_FIELD = new ParseField("grid_type");
private static final ParseField EXTENT_FIELD = new ParseField("extent");
private static final ParseField EXACT_BOUNDS_FIELD = new ParseField("exact_bounds");

protected enum GRID_TYPE {
GRID, POINT;

static GRID_TYPE fromString(String type) {
switch (type) {
case "grid" : return GRID;
case "point" : return POINT;
default: throw new IllegalArgumentException("Invalid grid type [" + type + "]");
}
}
}

protected final ObjectParser<R, RestRequest> parser;

Expand All @@ -56,12 +81,20 @@ protected interface ResponseBuilder {
private final Supplier<R> emptyRequestProvider;

protected static class Request {
QueryBuilder queryBuilder;
String index;
String field;
int x;
int y;
int z;
private QueryBuilder queryBuilder;
private String index;
private String field;
private int x;
private int y;
private int z;
private Map<String, Object> runtimeMappings = emptyMap();
private int gridPrecision = 8;
private GRID_TYPE gridType = GRID_TYPE.GRID;
private int size = 10000;
private int extent = 4096;
private AggregatorFactories.Builder aggBuilder;
private List<FieldAndFormat> fields = emptyList();
private boolean exact_bounds;

public String getIndex() {
return index;
Expand Down Expand Up @@ -103,11 +136,37 @@ public void setZ(int z) {
this.z = z;
}

public int getExtent() {
return extent;
}

public void setExtent(int extent) {
// TODO: validation
this.extent = extent;
}

public boolean getExactBounds() {
return exact_bounds;
}

public void setExactBounds(boolean exact_bounds) {
this.exact_bounds = exact_bounds;
}

public List<FieldAndFormat> getFields() {
return fields;
}

public void setFields(List<FieldAndFormat> fields) {
this.fields = fields;
}

public QueryBuilder getQueryBuilder() {
return queryBuilder;
}

public void setQueryBuilder(QueryBuilder queryBuilder) {
// TODO: validation
this.queryBuilder = queryBuilder;
}

Expand All @@ -125,17 +184,94 @@ public QueryBuilder getQuery() throws IOException {
}
return qBuilder;
}

public Map<String, Object> getRuntimeMappings() {
return runtimeMappings;
}

public void setRuntimeMappings(Map<String, Object> runtimeMappings) {
this.runtimeMappings = runtimeMappings;
}

public int getGridPrecision() {
return gridPrecision;
}

public void setGridPrecision(int gridPrecision) {
if (gridPrecision < 0 || gridPrecision > 8) {
throw new IllegalArgumentException("Invalid grid precision, value should be between 0 and 8, got [" + gridPrecision + "]");
}
this.gridPrecision = gridPrecision;
}

public GRID_TYPE getGridType() {
return gridType;
}

public void setGridType(String gridType) {
this.gridType = GRID_TYPE.fromString(gridType);
}

public int getSize() {
return size;
}

public void setSize(int size) {
// TODO: validation
this.size = size;
}

public AggregatorFactories.Builder getAggBuilder() {
return aggBuilder;
}

public void setAggBuilder(AggregatorFactories.Builder aggBuilder) {
// TODO: validation
this.aggBuilder = aggBuilder;
}
}

protected AbstractVectorTileSearchAction(Supplier<R> emptyRequestProvider) {
this.emptyRequestProvider = emptyRequestProvider;
parser = new ObjectParser<>(getName(), emptyRequestProvider);
parser.declareInt(Request::setSize, SearchSourceBuilder.SIZE_FIELD);
parser.declareField(
Request::setFields,
AbstractVectorTileSearchAction::parseFetchFields,
SearchSourceBuilder.FETCH_FIELDS_FIELD,
ObjectParser.ValueType.OBJECT_ARRAY
);
parser.declareField(
Request::setQueryBuilder,
(CheckedFunction<XContentParser, QueryBuilder, IOException>) AbstractQueryBuilder::parseInnerQueryBuilder,
SearchSourceBuilder.QUERY_FIELD,
ObjectParser.ValueType.OBJECT
);
parser.declareField(
Request::setRuntimeMappings,
XContentParser::map,
SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD,
ObjectParser.ValueType.OBJECT
);
parser.declareField(
Request::setAggBuilder,
AggregatorFactories::parseAggregators,
SearchSourceBuilder.AGGS_FIELD,
ObjectParser.ValueType.OBJECT
);
// Specific for vector tiles
parser.declareInt(Request::setGridPrecision, GRID_PRECISION_FIELD);
parser.declareInt(Request::setExtent, EXTENT_FIELD);
parser.declareBoolean(Request::setExactBounds, EXACT_BOUNDS_FIELD);
parser.declareString(Request::setGridType, GRID_TYPE_FIELD);
}

private static List<FieldAndFormat> parseFetchFields(XContentParser parser) throws IOException {
List<FieldAndFormat> fetchFields = new ArrayList<>();
while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) {
fetchFields.add(FieldAndFormat.fromXContent(parser));
}
return fetchFields;
}

protected abstract ResponseBuilder doParseRequest(RestRequest restRequest, R request, SearchRequestBuilder searchRequestBuilder)
Expand All @@ -159,7 +295,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient

SearchRequestBuilder searchRequestBuilder = client.prepareSearch(Strings.splitStringByCommaToArray(request.getIndex()));
searchRequestBuilder.setQuery(request.getQuery());
searchRequestBuilder.setSize(0);
ResponseBuilder responseBuilder = doParseRequest(restRequest, request, searchRequestBuilder);

// TODO: how do we handle cancellations?
Expand Down
10 changes: 6 additions & 4 deletions ...ugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@
import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter;
import com.wdtinc.mapbox_vector_tile.adapt.jts.JtsAdapter;
import com.wdtinc.mapbox_vector_tile.adapt.jts.TileGeomResult;
import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter;
import com.wdtinc.mapbox_vector_tile.build.MvtLayerParams;
import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps;
import org.elasticsearch.geometry.Circle;
Expand All @@ -37,7 +36,6 @@
public class FeatureFactory {

private final IGeometryFilter acceptAllGeomFilter = geometry -> true;
private final IUserDataConverter ignoreUserData = new UserDataIgnoreConverter();
private final MvtLayerParams layerParams;
private final GeometryFactory geomFactory = new GeometryFactory();
private final MvtLayerProps layerProps = new MvtLayerProps();
Expand All @@ -55,12 +53,16 @@ public FeatureFactory(int z, int x, int y, int extent) {
this.layerParams = new MvtLayerParams(extent, extent);
}

public List<VectorTile.Tile.Feature> getFeatures(Geometry geometry) {
public List<VectorTile.Tile.Feature> getFeatures(Geometry geometry, IUserDataConverter userData) {
TileGeomResult tileGeom =
JtsAdapter.createTileGeom(JtsAdapter.flatFeatureList(geometry.visit(builder)),
tileEnvelope, clipEnvelope, geomFactory, layerParams, acceptAllGeomFilter);
// MVT tile geometry to MVT features
return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, ignoreUserData);
return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, userData);
}

public MvtLayerProps getLayerProps() {
return layerProps;
}

private static class JTSGeometryBuilder implements GeometryVisitor<org.locationtech.jts.geom.Geometry, IllegalArgumentException> {
Expand Down
Loading