Add 'mvt' field type format to geo fields

server/src/main/java/org/elasticsearch/common/geo/GeoFormatterFactory.java

@@ -0,0 +1,70 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.common.geo;
+
+import org.elasticsearch.geometry.Geometry;
+import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
+
+import java.util.List;
+import java.util.Locale;
+import java.util.function.Function;
+
+/**
+ * Output formatters for geo fields. Adds support for vector tiles.
+ */
+public class GeoFormatterFactory {
+
+    @FunctionalInterface
+    public interface VectorTileEngine<T>  {
+        /**
+         * Returns a formatter for a specific tile.
+         */
+        Function<List<T>, List<Object>> getFormatter(int z, int x, int y, int extent);
+    }
+
+    private static final String MVT = "mvt";
+
+    /**
+     * Returns a formatter by name
+     */
+    public static <T> Function<List<T>, List<Object>> getFormatter(String format, Function<T, Geometry> toGeometry,
+                                                                          VectorTileEngine<T> mvt) {
+        final int start = format.indexOf('(');
+        if (start == -1)  {
+            return GeometryFormatterFactory.getFormatter(format, toGeometry);
+        }
+        final String formatName = format.substring(0, start);
+        if (MVT.equals(formatName) == false) {
+            throw new IllegalArgumentException("Invalid format: " + formatName);
+        }
+        final String param = format.substring(start + 1, format.length() - 1);
+        // we expect either z/x/y or z/x/y@extent
+        final String[] parts = param.split("@", 3);
+        if (parts.length > 2) {
+            throw new IllegalArgumentException(
+                "Invalid mvt formatter parameter [" + param + "]. Must have the form \"zoom/x/y\" or \"zoom/x/y@extent\"."
+            );
+        }
+        final int extent = parts.length == 2 ? Integer.parseInt(parts[1]) : 4096;
+        final String[] tileBits = parts[0].split("/", 4);
+        if (tileBits.length != 3) {
+            throw new IllegalArgumentException(
+                "Invalid tile string [" + parts[0] + "]. Must be three integers in a form \"zoom/x/y\"."
+            );
+        }
+        final int z = GeoTileUtils.checkPrecisionRange(Integer.parseInt(tileBits[0]));
+        final int tiles = 1 << z;
+        final int x = Integer.parseInt(tileBits[1]);
+        final int y = Integer.parseInt(tileBits[2]);
+        if (x < 0 || y < 0 || x >= tiles || y >= tiles) {
+            throw new IllegalArgumentException(String.format(Locale.ROOT, "Zoom/X/Y combination is not valid: %d/%d/%d", z, x, y));
+        }
+        return mvt.getFormatter(z, x, y, extent);
+    }
+}

x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java → server/src/main/java/org/elasticsearch/common/geo/SimpleFeatureFactory.java

@@ -1,25 +1,25 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-package org.elasticsearch.xpack.vectortile.feature;
+package org.elasticsearch.common.geo;
 
 import org.apache.lucene.util.BitUtil;
 import org.elasticsearch.common.io.stream.BytesStreamOutput;
-import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
 
 import java.io.IOException;
+import java.io.UncheckedIOException;
 import java.util.Comparator;
 import java.util.List;
 
 /**
- * Similar to {@link FeatureFactory} but only supports points and rectangles. It is just
- * more efficient for those shapes and it does not use external dependencies.
+ * Transforms points and rectangles objects in WGS84 into mvt features.
  */
 public class SimpleFeatureFactory {
 
@@ -63,11 +63,11 @@ public byte[] point(double lon, double lat) throws IOException {
     /**
      * Returns a {@code byte[]} containing the mvt representation of the provided points
      */
-    public byte[] points(List<Point> multiPoint) throws IOException {
-        multiPoint.sort(Comparator.comparingDouble(Point::getLon).thenComparingDouble(Point::getLat));
+    public byte[] points(List<GeoPoint> multiPoint) {
+        multiPoint.sort(Comparator.comparingDouble(GeoPoint::getLon).thenComparingDouble(GeoPoint::getLat));
         final int[] commands = new int[2 * multiPoint.size() + 1];
         int pos = 1, prevLon = 0, prevLat = 0, numPoints = 0;
-        for (Point point : multiPoint) {
+        for (GeoPoint point : multiPoint) {
             final int posLon = lon(point.getLon());
             if (posLon > extent || posLon < 0) {
                 continue;
@@ -89,7 +89,11 @@ public byte[] points(List<Point> multiPoint) throws IOException {
             return EMPTY;
         }
         commands[0] = encodeCommand(MOVETO, numPoints);
-        return writeCommands(commands, 1, pos);
+        try {
+            return writeCommands(commands, 1, pos);
+        } catch (IOException ioe) {
+            throw new UncheckedIOException(ioe);
+        }
     }
 
     /**

x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SphericalMercatorUtils.java → server/src/main/java/org/elasticsearch/common/geo/SphericalMercatorUtils.java

@@ -1,20 +1,21 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-package org.elasticsearch.xpack.vectortile.feature;
+package org.elasticsearch.common.geo;
 
 import org.elasticsearch.geometry.Rectangle;
 
 /**
  * Utility functions to transforms WGS84 coordinates into spherical mercator.
  */
-class SphericalMercatorUtils {
+public class SphericalMercatorUtils {
 
-    private static double MERCATOR_FACTOR = 20037508.34 / 180.0;
+    private static final double MERCATOR_FACTOR = 20037508.34 / 180.0;
 
     /**
      * Transforms WGS84 longitude to a Spherical mercator longitude

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -19,11 +19,13 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.common.CheckedBiFunction;
 import org.elasticsearch.common.Explicit;
+import org.elasticsearch.common.geo.GeoFormatterFactory;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoShapeUtils;
 import org.elasticsearch.common.geo.GeoUtils;
 import org.elasticsearch.common.geo.GeometryFormatterFactory;
 import org.elasticsearch.common.geo.ShapeRelation;
+import org.elasticsearch.common.geo.SimpleFeatureFactory;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.support.MapXContentParser;
@@ -243,7 +245,11 @@ public String typeName() {
 
         @Override
         protected  Function<List<GeoPoint>, List<Object>> getFormatter(String format) {
-            return GeometryFormatterFactory.getFormatter(format, p -> new Point(p.lon(), p.lat()));
+            return GeoFormatterFactory.getFormatter(format, p -> new Point(p.getLon(), p.getLat()),
+                (z, x, y, extent) -> {
+                    final SimpleFeatureFactory featureFactory = new SimpleFeatureFactory(z, x, y, extent);
+                    return points -> List.of(featureFactory.points(points));
+                });
         }
 
         @Override

x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactoryTests.java → server/src/test/java/org/elasticsearch/common/geo/SimpleFeatureFactoryTests.java

@@ -1,14 +1,14 @@
 /*
  * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
  * or more contributor license agreements. Licensed under the Elastic License
- * 2.0; you may not use this file except in compliance with the Elastic License
- * 2.0.
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
  */
 
-package org.elasticsearch.xpack.vectortile.feature;
+package org.elasticsearch.common.geo;
 
 import org.apache.lucene.geo.GeoTestUtil;
-import org.elasticsearch.geometry.Point;
 import org.elasticsearch.geometry.Rectangle;
 import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils;
 import org.elasticsearch.test.ESTestCase;
@@ -48,7 +48,7 @@ public void testPoint() throws IOException {
         }
     }
 
-    public void testMultiPoint() throws IOException {
+    public void testMultiPoint() {
         int z = randomIntBetween(3, 10);
         int x = randomIntBetween(0, (1 << z) - 1);
         int y = randomIntBetween(0, (1 << z) - 1);
@@ -57,20 +57,20 @@ public void testMultiPoint() throws IOException {
         Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z);
         int numPoints = randomIntBetween(2, 10);
         {
-            List<Point> points = new ArrayList<>();
+            List<GeoPoint> points = new ArrayList<>();
             double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() >= l || rectangle.getMaxY() <= l, GeoTestUtil::nextLatitude);
             double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() >= l || rectangle.getMaxX() <= l, GeoTestUtil::nextLongitude);
-            points.add(new Point(lon, lat));
+            points.add(new GeoPoint(lat, lon));
             for (int i = 0; i < numPoints - 1; i++) {
-                points.add(new Point(GeoTestUtil.nextLongitude(), GeoTestUtil.nextLatitude()));
+                points.add(new GeoPoint(GeoTestUtil.nextLatitude(), GeoTestUtil.nextLongitude()));
             }
             assertThat(builder.points(points).length, Matchers.greaterThan(0));
         }
         {
             int xNew = randomValueOtherThanMany(v -> Math.abs(v - x) < 2, () -> randomIntBetween(0, (1 << z) - 1));
             int yNew = randomValueOtherThanMany(v -> Math.abs(v - y) < 2, () -> randomIntBetween(0, (1 << z) - 1));
             Rectangle rectangleNew = GeoTileUtils.toBoundingBox(xNew, yNew, z);
-            List<Point> points = new ArrayList<>();
+            List<GeoPoint> points = new ArrayList<>();
             for (int i = 0; i < numPoints; i++) {
                 double lat = randomValueOtherThanMany(
                     (l) -> rectangleNew.getMinY() >= l || rectangleNew.getMaxY() <= l,
@@ -80,7 +80,7 @@ public void testMultiPoint() throws IOException {
                     (l) -> rectangleNew.getMinX() >= l || rectangleNew.getMaxX() <= l,
                     GeoTestUtil::nextLongitude
                 );
-                points.add(new Point(lon, lat));
+                points.add(new GeoPoint(lat, lon));
             }
             assertThat(builder.points(points).length, Matchers.equalTo(0));
         }
@@ -95,24 +95,24 @@ public void testPointsMethodConsistency() throws IOException {
         Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z);
         int extraPoints = randomIntBetween(1, 10);
         {
-            List<Point> points = new ArrayList<>();
+            List<GeoPoint> points = new ArrayList<>();
             double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() > l || rectangle.getMaxY() < l, GeoTestUtil::nextLatitude);
             double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() > l || rectangle.getMaxX() < l, GeoTestUtil::nextLongitude);
-            points.add(new Point(lon, lat));
+            points.add(new GeoPoint(lat, lon));
             assertArrayEquals(builder.points(points), builder.point(lon, lat));
             for (int i = 0; i < extraPoints; i++) {
-                points.add(new Point(lon, lat));
+                points.add(new GeoPoint(lat, lon));
             }
             assertArrayEquals(builder.points(points), builder.point(lon, lat));
         }
         {
-            List<Point> points = new ArrayList<>();
+            List<GeoPoint> points = new ArrayList<>();
             double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() <= l && rectangle.getMaxY() >= l, GeoTestUtil::nextLatitude);
             double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() <= l && rectangle.getMaxX() >= l, GeoTestUtil::nextLongitude);
-            points.add(new Point(lon, lat));
+            points.add(new GeoPoint(lat, lon));
             assertArrayEquals(builder.points(points), builder.point(lon, lat));
             for (int i = 0; i < extraPoints; i++) {
-                points.add(new Point(lon, lat));
+                points.add(new GeoPoint(lat, lon));
             }
             assertArrayEquals(builder.points(points), builder.point(lon, lat));
         }

server/src/test/java/org/elasticsearch/index/mapper/GeoPointFieldTypeTests.java

@@ -8,9 +8,14 @@
 
 package org.elasticsearch.index.mapper;
 
+import org.apache.lucene.geo.GeoTestUtil;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.SimpleFeatureFactory;
 import org.elasticsearch.script.ScriptCompiler;
+import org.hamcrest.Matchers;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 
@@ -54,4 +59,36 @@ public void testFetchSourceValue() throws IOException {
         sourceValue = "malformed";
         assertEquals(List.of(), fetchSourceValue(mapper, sourceValue, null));
     }
+
+    public void testFetchVectorTile() throws IOException {
+        MappedFieldType mapper
+            = new GeoPointFieldMapper.Builder("field", ScriptCompiler.NONE, false).build(new ContentPath()).fieldType();
+        final int z = randomIntBetween(1, 10);
+        int x = randomIntBetween(0, (1 << z) - 1);
+        int y = randomIntBetween(0, (1 << z) - 1);
+        final SimpleFeatureFactory featureFactory;
+        final String mvtString;
+        if (randomBoolean()) {
+            int extent = randomIntBetween(1 << 8, 1 << 14);
+            mvtString = "mvt(" + z + "/" + x + "/" + y + "@" + extent + ")";
+            featureFactory = new SimpleFeatureFactory(z, x, y, extent);
+        } else {
+            mvtString = "mvt(" + z + "/" + x + "/" + y + ")";
+            featureFactory = new SimpleFeatureFactory(z, x, y, 4096);
+        }
+        List<GeoPoint> geoPoints = new ArrayList<>();
+        List<List<Double>> values = new ArrayList<>();
+        for (int i = 0; i < randomIntBetween(1, 10); i++) {
+            final double lat = GeoTestUtil.nextLatitude();
+            final double lon = GeoTestUtil.nextLongitude();
+            List<?> sourceValue = fetchSourceValue(mapper, List.of(lon, lat), mvtString);
+            assertThat(sourceValue.size(), Matchers.equalTo(1));
+            assertThat(sourceValue.get(0), Matchers.equalTo(featureFactory.point(lon, lat)));
+            geoPoints.add(new GeoPoint(lat, lon));
+            values.add(List.of(lon, lat));
+        }
+        List<?> sourceValue = fetchSourceValue(mapper, values, mvtString);
+        assertThat(sourceValue.size(), Matchers.equalTo(1));
+        assertThat(sourceValue.get(0), Matchers.equalTo(featureFactory.points(geoPoints)));
+    }
 }

x-pack/plugin/spatial/build.gradle

@@ -13,6 +13,7 @@ esplugin {
 dependencies {
   compileOnly project(path: xpackModule('core'))
   testImplementation(testArtifact(project(xpackModule('core'))))
+  testImplementation project(path: xpackModule('vector-tile'))
   yamlRestTestImplementation(testArtifact(project(xpackModule('core'))))
   api project(path: ':modules:geo')
   restTestConfig project(path: ':modules:geo', configuration: 'restTests')

x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java

@@ -6,6 +6,7 @@
  */
 package org.elasticsearch.xpack.spatial;
 
+import org.apache.lucene.util.SetOnce;
 import org.elasticsearch.action.ActionRequest;
 import org.elasticsearch.action.ActionResponse;
 import org.elasticsearch.common.xcontent.ContextParser;
@@ -15,6 +16,7 @@
 import org.elasticsearch.license.LicenseUtils;
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
+import org.elasticsearch.plugins.ExtensiblePlugin;
 import org.elasticsearch.plugins.IngestPlugin;
 import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.plugins.SearchPlugin;
@@ -63,14 +65,17 @@
 
 import static java.util.Collections.singletonList;
 
-public class SpatialPlugin extends GeoPlugin implements ActionPlugin, MapperPlugin, SearchPlugin, IngestPlugin {
+public class SpatialPlugin extends GeoPlugin implements ActionPlugin, MapperPlugin, SearchPlugin, IngestPlugin, ExtensiblePlugin {
    private final SpatialUsage usage = new SpatialUsage();
 
     // to be overriden by tests
     protected XPackLicenseState getLicenseState() {
         return XPackPlugin.getSharedLicenseState();
     }
 
+    // register the vector tile factory from a different module
+    private final SetOnce<VectorTileExtension> vectorTileExtension = new SetOnce<>();
+
     @Override
     public List<ActionPlugin.ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(
@@ -84,7 +89,7 @@ public Map<String, Mapper.TypeParser> getMappers() {
         Map<String, Mapper.TypeParser> mappers = new HashMap<>(super.getMappers());
         mappers.put(ShapeFieldMapper.CONTENT_TYPE, ShapeFieldMapper.PARSER);
         mappers.put(PointFieldMapper.CONTENT_TYPE, PointFieldMapper.PARSER);
-        mappers.put(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, GeoShapeWithDocValuesFieldMapper.PARSER);
+        mappers.put(GeoShapeWithDocValuesFieldMapper.CONTENT_TYPE, new GeoShapeWithDocValuesFieldMapper.TypeParser(vectorTileExtension));
         return Collections.unmodifiableMap(mappers);
     }
 
@@ -205,4 +210,10 @@ private <T> ContextParser<String, T> checkLicense(ContextParser<String, T> realP
             return realParser.parse(parser, name);
         };
     }
+
+    @Override
+    public void loadExtensions(ExtensionLoader loader) {
+        // we only expect one vector tile extension that comes from the vector tile module.
+        loader.loadExtensions(VectorTileExtension.class).forEach(vectorTileExtension::set);
+    }
 }