diff --git a/server/src/main/java/org/elasticsearch/common/util/Maps.java b/server/src/main/java/org/elasticsearch/common/util/Maps.java index 6336522d14272..473723b4b6fbb 100644 --- a/server/src/main/java/org/elasticsearch/common/util/Maps.java +++ b/server/src/main/java/org/elasticsearch/common/util/Maps.java @@ -10,9 +10,11 @@ import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.TreeMap; import java.util.stream.Collectors; public class Maps { @@ -76,4 +78,54 @@ private static boolean checkIsImmutableMap(final Map map, final K k } return true; } + + /** + * Returns an array where all internal maps and optionally arrays are flattened into the root map. + * + * For example the map {"foo": {"bar": 1, "baz": [2, 3]}} will become {"foo.bar": 1, "foo.baz.0": 2, "foo.baz.1": 3}. Note that if + * maps contains keys with "." or numbers it is possible that such keys will be silently overridden. For example the map + * {"foo": {"bar": 1}, "foo.bar": 2} will become {"foo.bar": 1} or {"foo.bar": 2}. + * + * @param map - input to be flattened + * @param flattenArrays - if false, arrays will be ignored + * @param ordered - if true the resulted map will be sorted + * @return + */ + public static Map flatten(Map map, boolean flattenArrays, boolean ordered) { + return flatten(map, flattenArrays, ordered, null); + } + + @SuppressWarnings("unchecked") + private static Map flatten(Map map, boolean flattenArrays, boolean ordered, String parentPath) { + Map flatMap = ordered ? new TreeMap<>() : new HashMap<>(); + String prefix = parentPath != null ? parentPath + "." : ""; + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() instanceof Map) { + flatMap.putAll(flatten((Map) entry.getValue(), flattenArrays, ordered, prefix + entry.getKey())); + } else if (flattenArrays && entry.getValue() instanceof List) { + flatMap.putAll(flatten((List) entry.getValue(), ordered, prefix + entry.getKey())); + } else { + flatMap.put(prefix + entry.getKey(), entry.getValue()); + } + } + return flatMap; + } + + @SuppressWarnings("unchecked") + private static Map flatten(List list, boolean ordered, String parentPath) { + Map flatMap = ordered ? new TreeMap<>() : new HashMap<>(); + String prefix = parentPath != null ? parentPath + "." : ""; + for (int i = 0; i < list.size(); i++) { + Object cur = list.get(i); + if (cur instanceof Map) { + flatMap.putAll(flatten((Map) cur, true, ordered, prefix + i)); + } + if (cur instanceof List) { + flatMap.putAll(flatten((List) cur, ordered, prefix + i)); + } else { + flatMap.put(prefix + i, cur); + } + } + return flatMap; + } } diff --git a/server/src/test/java/org/elasticsearch/common/util/MapsTests.java b/server/src/test/java/org/elasticsearch/common/util/MapsTests.java index 1c114f1f8bd45..e3ce2e7859a90 100644 --- a/server/src/test/java/org/elasticsearch/common/util/MapsTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/MapsTests.java @@ -11,12 +11,17 @@ import org.elasticsearch.test.ESTestCase; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.function.Supplier; +import java.util.stream.Collectors; import java.util.stream.IntStream; import static java.util.stream.Collectors.toMap; +import static org.hamcrest.Matchers.equalTo; + public class MapsTests extends ESTestCase { @@ -50,4 +55,68 @@ private static Map randomMap(int size, Supplier keyGenerator, Su IntStream.range(0, size).forEach(i -> map.put(keyGenerator.get(), valueGenerator.get())); return map; } + + public void testFlatten() { + Map map = randomNestedMap(10); + Map flatten = Maps.flatten(map, true, true); + assertThat(flatten.size(), equalTo(deepCount(map.values()))); + for (Map.Entry entry : flatten.entrySet()) { + assertThat(entry.getKey(), entry.getValue(), equalTo(deepGet(entry.getKey(), map))); + } + } + + @SuppressWarnings("unchecked") + private static Object deepGet(String path, Object obj) { + Object cur = obj; + String[] keys = path.split("\\."); + for (String key : keys) { + if (Character.isDigit(key.charAt(0))) { + List list = (List) cur; + cur = list.get(Integer.parseInt(key)); + } else { + Map map = (Map) cur; + cur = map.get(key); + } + } + return cur; + } + + @SuppressWarnings("unchecked") + private int deepCount(Collection map) { + int sum = 0; + for (Object val : map) { + if (val instanceof Map) { + sum += deepCount(((Map) val).values()); + } else if (val instanceof List) { + sum += deepCount((List) val); + } else { + sum ++; + } + } + return sum; + } + + private Map randomNestedMap(int level) { + final Supplier keyGenerator = () -> randomAlphaOfLengthBetween(1, 5); + final Supplier arrayValueGenerator = () -> random().ints(randomInt(5)) + .boxed() + .map(s -> (Object) s) + .collect(Collectors.toList()); + + final Supplier mapSupplier; + if (level > 0) { + mapSupplier = () -> randomNestedMap(level - 1); + } else { + mapSupplier = ESTestCase::randomLong; + } + final Supplier> valueSupplier = () -> randomFrom( + ESTestCase::randomBoolean, + ESTestCase::randomDouble, + ESTestCase::randomLong, + arrayValueGenerator, + mapSupplier + ); + final Supplier valueGenerator = () -> valueSupplier.get().get(); + return randomMap(randomInt(5), keyGenerator, valueGenerator); + } } diff --git a/x-pack/plugin/vector-tile/build.gradle b/x-pack/plugin/vector-tile/build.gradle new file mode 100644 index 0000000000000..337d4986cbf1d --- /dev/null +++ b/x-pack/plugin/vector-tile/build.gradle @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import org.elasticsearch.gradle.internal.info.BuildParams + +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.java-rest-test' + +esplugin { + name 'vector-tile' + description 'A plugin for mapbox vector tile features' + classname 'org.elasticsearch.xpack.vectortile.VectorTilePlugin' + extendedPlugins = ['x-pack-core'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + testImplementation(testArtifact(project(xpackModule('core')))) + api "com.wdtinc:mapbox-vector-tile:3.1.0" + api "com.google.protobuf:protobuf-java:3.14.0" + javaRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0") + javaRestTestImplementation("com.google.protobuf:protobuf-java:3.14.0") +} + +testClusters.all { + setting 'xpack.license.self_generated.type', 'trial' + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'false' + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.vector_tile_feature_flag_registered', 'true' + } +} + +tasks.named("test").configure { + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.vector_tile_feature_flag_registered', 'true' + } +} + +tasks.named("thirdPartyAudit").configure { + ignoreViolations( + // uses internal java api: sun.misc.Unsafe + 'com.google.protobuf.UnsafeUtil', + 'com.google.protobuf.MessageSchema', + 'com.google.protobuf.UnsafeUtil$1', + 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', + 'com.google.protobuf.UnsafeUtil$MemoryAccessor' + ) + + ignoreMissingClasses( + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory' + ) +} + diff --git a/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-3.1.0.jar.sha1 b/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-3.1.0.jar.sha1 new file mode 100644 index 0000000000000..c98d2861a1bb8 --- /dev/null +++ b/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-3.1.0.jar.sha1 @@ -0,0 +1 @@ +06c4432c7885a3938571a57e73cc1444d7a39f12 \ No newline at end of file diff --git a/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-LICENSE.txt b/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-LICENSE.txt new file mode 100644 index 0000000000000..f433b1a53f5b8 --- /dev/null +++ b/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-LICENSE.txt @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-NOTICE.txt b/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/vector-tile/licenses/protobuf-java-3.14.0.jar.sha1 b/x-pack/plugin/vector-tile/licenses/protobuf-java-3.14.0.jar.sha1 new file mode 100644 index 0000000000000..a8a1dde74b47d --- /dev/null +++ b/x-pack/plugin/vector-tile/licenses/protobuf-java-3.14.0.jar.sha1 @@ -0,0 +1 @@ +bb6430f70647fc349fffd1690ddb889dc3ea6699 \ No newline at end of file diff --git a/x-pack/plugin/vector-tile/licenses/protobuf-java-LICENSE.txt b/x-pack/plugin/vector-tile/licenses/protobuf-java-LICENSE.txt new file mode 100644 index 0000000000000..fc4865a95ffdc --- /dev/null +++ b/x-pack/plugin/vector-tile/licenses/protobuf-java-LICENSE.txt @@ -0,0 +1,33 @@ + +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/x-pack/plugin/vector-tile/licenses/protobuf-java-NOTICE.txt b/x-pack/plugin/vector-tile/licenses/protobuf-java-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java new file mode 100644 index 0000000000000..390f4dd81fa60 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java @@ -0,0 +1,397 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile; + +import com.wdtinc.mapbox_vector_tile.VectorTile; + +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; + +/** + * Rest test for _mvt end point. The test only check that the structure of the vector tiles is sound in + * respect to the number of layers returned and the number of features abd tags in each layer. + */ +public class VectorTileRestIT extends ESRestTestCase { + + private static final String INDEX_POINTS = "index-points"; + private static final String INDEX_SHAPES = "index-shapes"; + private static final String INDEX_ALL = "index*"; + private static final String META_LAYER = "meta"; + private static final String HITS_LAYER = "hits"; + private static final String AGGS_LAYER = "aggs"; + + private int x, y, z; + + @Before + public void indexDocuments() throws IOException { + z = randomIntBetween(1, GeoTileUtils.MAX_ZOOM - 10); + x = randomIntBetween(0, (1 << z) - 1); + y = randomIntBetween(0, (1 << z) - 1); + indexPoints(); + indexShapes(); + } + + private void indexPoints() throws IOException { + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS); + Response response = client().performRequest(createRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS + "/_mapping"); + mappingRequest.setJsonEntity( + "{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_point\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + "}" + ); + response = client().performRequest(mappingRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + double x = (r.getMaxX() + r.getMinX()) / 2; + double y = (r.getMaxY() + r.getMinY()) / 2; + for (int i = 0; i < 30; i += 10) { + for (int j = 0; j <= i; j++) { + final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_doc"); + putRequest.setJsonEntity( + "{\n" + + " \"location\": \"POINT(" + + x + + " " + + y + + ")\", \"name\": \"point" + + i + + "\"" + + ", \"value1\": " + + i + + ", \"value2\": " + + (i + 1) + + "\n" + + "}" + ); + response = client().performRequest(putRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + } + } + + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_refresh"); + response = client().performRequest(flushRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } + + private void indexShapes() throws IOException { + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES); + Response response = client().performRequest(createRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES + "/_mapping"); + mappingRequest.setJsonEntity( + "{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_shape\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + "}" + ); + response = client().performRequest(mappingRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + + final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_doc"); + putRequest.setJsonEntity( + "{\n" + + " \"location\": \"BBOX (" + + r.getMinLon() + + ", " + + r.getMaxLon() + + "," + + r.getMaxLat() + + "," + + r.getMinLat() + + ")\"" + + ", \"name\": \"rectangle\"" + + ", \"value1\": " + + 1 + + ", \"value2\": " + + 2 + + "\n" + + "}" + ); + response = client().performRequest(putRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_refresh"); + response = client().performRequest(flushRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } + + @After + public void deleteData() throws IOException { + final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_POINTS); + final Response response = client().performRequest(deleteRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } + + public void testBasicGet() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\" : 100}"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 14); + } + + public void testExtent() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\" : 100, \"extent\" : 256}"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 256, 33, 1); + assertLayer(tile, AGGS_LAYER, 256, 1, 1); + assertLayer(tile, META_LAYER, 256, 1, 14); + } + + public void testEmpty() throws Exception { + final int newY = (1 << z) - 1 == y ? y - 1 : y + 1; + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(1)); + assertLayer(tile, META_LAYER, 4096, 1, 10); + } + + public void testGridPrecision() throws Exception { + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_precision\": 7 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 14); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_precision\": 9 }"); + final ResponseException ex = expectThrows(ResponseException.class, () -> execute(mvtRequest)); + assertThat(ex.getResponse().getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_BAD_REQUEST)); + } + } + + public void testGridType() throws Exception { + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_type\": \"point\" }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 14); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_type\": \"grid\" }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 14); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_type\": \"invalid_type\" }"); + final ResponseException ex = expectThrows(ResponseException.class, () -> execute(mvtRequest)); + assertThat(ex.getResponse().getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_BAD_REQUEST)); + } + } + + public void testNoAggLayer() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_precision\": 0 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, META_LAYER, 4096, 1, 9); + } + + public void testNoHitsLayer() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\": 0 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 13); + } + + public void testRuntimeFieldWithSort() throws Exception { + String runtimeMapping = "\"runtime_mappings\": {\n" + + " \"width\": {\n" + + " \"script\": " + + "\"emit(doc['location'].getBoundingBox().bottomRight().getLon() - doc['location'].getBoundingBox().topLeft().getLon())\",\n" + + " \"type\": \"double\"\n" + + " }\n" + + "}\n"; + { + // desc order, polygon should be the first hit + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_ALL + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity( + "{\n" + + " \"size\" : 100,\n" + + " \"grid_precision\" : 0,\n" + + runtimeMapping + + "," + + " \"sort\" : [\n" + + " {\n" + + " \"width\": {\n" + + " \"order\": \"desc\"\n" + + " }\n" + + " }\n" + + " ]" + + "}" + ); + + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, HITS_LAYER, 4096, 34, 1); + final VectorTile.Tile.Layer layer = getLayer(tile, HITS_LAYER); + assertThat(layer.getFeatures(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POLYGON)); + assertLayer(tile, META_LAYER, 4096, 1, 8); + } + { + // asc order, polygon should be the last hit + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_ALL + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity( + "{\n" + + " \"size\" : 100,\n" + + " \"grid_precision\" : 0,\n" + + runtimeMapping + + "," + + " \"sort\" : [\n" + + " {\n" + + " \"width\": {\n" + + " \"order\": \"asc\"\n" + + " }\n" + + " }\n" + + " ]" + + "}" + ); + + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, HITS_LAYER, 4096, 34, 1); + final VectorTile.Tile.Layer layer = getLayer(tile, HITS_LAYER); + assertThat(layer.getFeatures(33).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POLYGON)); + assertLayer(tile, META_LAYER, 4096, 1, 8); + } + } + + public void testBasicQueryGet() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity( + "{\n" + + " \"query\": {\n" + + " \"term\": {\n" + + " \"name\": {\n" + + " \"value\": \"point0\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 14); + } + + public void testBasicShape() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/" + z + "/" + x + "/" + y); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); + assertLayer(tile, META_LAYER, 4096, 1, 14); + } + + public void testWithFields() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"fields\": [\"name\", \"value1\"] }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 3); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); + assertLayer(tile, META_LAYER, 4096, 1, 14); + } + + public void testMinAgg() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity( + "{\n" + + " \"aggs\": {\n" + + " \"minVal\": {\n" + + " \"min\": {\n" + + " \"field\": \"value1\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2); + assertLayer(tile, META_LAYER, 4096, 1, 19); + } + + private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) { + final VectorTile.Tile.Layer layer = getLayer(tile, name); + assertThat(layer.getExtent(), Matchers.equalTo(extent)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(numFeatures)); + assertThat(layer.getKeysCount(), Matchers.equalTo(numTags)); + } + + private VectorTile.Tile execute(Request mvtRequest) throws IOException { + final Response response = client().performRequest(mvtRequest); + final InputStream inputStream = response.getEntity().getContent(); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + return VectorTile.Tile.parseFrom(inputStream); + } + + private VectorTile.Tile.Layer getLayer(VectorTile.Tile tile, String layerName) { + for (int i = 0; i < tile.getLayersCount(); i++) { + final VectorTile.Tile.Layer layer = tile.getLayers(i); + if (layerName.equals(layer.getName())) { + return layer; + } + } + fail("Could not find layer " + layerName); + return null; + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java new file mode 100644 index 0000000000000..24fca7c3adc9f --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java @@ -0,0 +1,81 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.vectortile; + +import org.elasticsearch.Build; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.core.Booleans; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.threadpool.ExecutorBuilder; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.vectortile.rest.RestVectorTileAction; + +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +public class VectorTilePlugin extends Plugin implements ActionPlugin { + + // to be overriden by tests + protected XPackLicenseState getLicenseState() { + return XPackPlugin.getSharedLicenseState(); + } + + private static final Boolean VECTOR_TILE_FEATURE_FLAG_REGISTERED; + + static { + final String property = System.getProperty("es.vector_tile_feature_flag_registered"); + if (Build.CURRENT.isSnapshot() && property != null) { + throw new IllegalArgumentException("es.vector_tile_feature_flag_registered is only supported in non-snapshot builds"); + } + VECTOR_TILE_FEATURE_FLAG_REGISTERED = Booleans.parseBoolean(property, null); + } + + public boolean isVectorTileEnabled() { + return Build.CURRENT.isSnapshot() || (VECTOR_TILE_FEATURE_FLAG_REGISTERED != null && VECTOR_TILE_FEATURE_FLAG_REGISTERED); + } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + if (isVectorTileEnabled()) { + return org.elasticsearch.core.List.of(new RestVectorTileAction()); + } else { + return org.elasticsearch.core.List.of(); + } + } + + @Override + public List> getExecutorBuilders(Settings settings) { + FixedExecutorBuilder indexing = new FixedExecutorBuilder( + settings, + "vector_tile_generation", + 1, + -1, + "thread_pool.vectortile", + false + ); + return Collections.singletonList(indexing); + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java new file mode 100644 index 0000000000000..7f3d1884fe280 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java @@ -0,0 +1,196 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.feature; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IGeometryFilter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.JtsAdapter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.TileGeomResult; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerParams; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; + +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.LineString; + +import java.util.List; + +/** + * Transforms {@link Geometry} object in WGS84 into mvt features. + */ +public class FeatureFactory { + + private final IGeometryFilter acceptAllGeomFilter = geometry -> true; + private final MvtLayerParams layerParams; + private final GeometryFactory geomFactory = new GeometryFactory(); + private final MvtLayerProps layerProps = new MvtLayerProps(); + private final JTSGeometryBuilder builder; + + private final Envelope tileEnvelope; + private final Envelope clipEnvelope; + + public FeatureFactory(int z, int x, int y, int extent) { + this.tileEnvelope = FeatureFactoryUtils.getJTSTileBounds(z, x, y); + this.clipEnvelope = FeatureFactoryUtils.getJTSTileBounds(z, x, y); + this.clipEnvelope.expandBy(tileEnvelope.getWidth() * 0.1d, tileEnvelope.getHeight() * 0.1d); + this.builder = new JTSGeometryBuilder(geomFactory); + // TODO: Not sure what is the difference between extent and tile size? + this.layerParams = new MvtLayerParams(extent, extent); + } + + public List getFeatures(Geometry geometry, IUserDataConverter userData) { + TileGeomResult tileGeom = JtsAdapter.createTileGeom( + JtsAdapter.flatFeatureList(geometry.visit(builder)), + tileEnvelope, + clipEnvelope, + geomFactory, + layerParams, + acceptAllGeomFilter + ); + // MVT tile geometry to MVT features + return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, userData); + } + + public MvtLayerProps getLayerProps() { + return layerProps; + } + + private static class JTSGeometryBuilder implements GeometryVisitor { + + private final GeometryFactory geomFactory; + + JTSGeometryBuilder(GeometryFactory geomFactory) { + this.geomFactory = geomFactory; + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Circle circle) { + throw new IllegalArgumentException("Circle is not supported"); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(GeometryCollection collection) { + throw new IllegalArgumentException("Circle is not supported"); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(LinearRing ring) throws RuntimeException { + throw new IllegalArgumentException("LinearRing is not supported"); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Point point) throws RuntimeException { + return buildPoint(point); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(MultiPoint multiPoint) throws RuntimeException { + final org.locationtech.jts.geom.Point[] points = new org.locationtech.jts.geom.Point[multiPoint.size()]; + for (int i = 0; i < multiPoint.size(); i++) { + points[i] = buildPoint(multiPoint.get(i)); + } + return geomFactory.createMultiPoint(points); + } + + private org.locationtech.jts.geom.Point buildPoint(Point point) { + final double x = FeatureFactoryUtils.lonToSphericalMercator(point.getX()); + final double y = FeatureFactoryUtils.latToSphericalMercator(point.getY()); + return geomFactory.createPoint(new Coordinate(x, y)); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Line line) { + return buildLine(line); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(MultiLine multiLine) throws RuntimeException { + LineString[] lineStrings = new LineString[multiLine.size()]; + for (int i = 0; i < multiLine.size(); i++) { + lineStrings[i] = buildLine(multiLine.get(i)); + } + return geomFactory.createMultiLineString(lineStrings); + } + + private LineString buildLine(Line line) { + final Coordinate[] coordinates = new Coordinate[line.length()]; + for (int i = 0; i < line.length(); i++) { + final double x = FeatureFactoryUtils.lonToSphericalMercator(line.getX(i)); + final double y = FeatureFactoryUtils.latToSphericalMercator(line.getY(i)); + coordinates[i] = new Coordinate(x, y); + } + return geomFactory.createLineString(coordinates); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Polygon polygon) throws RuntimeException { + return buildPolygon(polygon); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(MultiPolygon multiPolygon) throws RuntimeException { + org.locationtech.jts.geom.Polygon[] polygons = new org.locationtech.jts.geom.Polygon[multiPolygon.size()]; + for (int i = 0; i < multiPolygon.size(); i++) { + polygons[i] = buildPolygon(multiPolygon.get(i)); + } + return geomFactory.createMultiPolygon(polygons); + } + + private org.locationtech.jts.geom.Polygon buildPolygon(Polygon polygon) { + final org.locationtech.jts.geom.LinearRing outerShell = buildLinearRing(polygon.getPolygon()); + if (polygon.getNumberOfHoles() == 0) { + return geomFactory.createPolygon(outerShell); + } + org.locationtech.jts.geom.LinearRing[] holes = new org.locationtech.jts.geom.LinearRing[polygon.getNumberOfHoles()]; + for (int i = 0; i < polygon.getNumberOfHoles(); i++) { + holes[i] = buildLinearRing(polygon.getHole(i)); + } + return geomFactory.createPolygon(outerShell, holes); + } + + private org.locationtech.jts.geom.LinearRing buildLinearRing(LinearRing ring) throws RuntimeException { + final Coordinate[] coordinates = new Coordinate[ring.length()]; + for (int i = 0; i < ring.length(); i++) { + final double x = FeatureFactoryUtils.lonToSphericalMercator(ring.getX(i)); + final double y = FeatureFactoryUtils.latToSphericalMercator(ring.getY(i)); + coordinates[i] = new Coordinate(x, y); + } + return geomFactory.createLinearRing(coordinates); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Rectangle rectangle) throws RuntimeException { + // TODO: handle degenerated rectangles? + final double xMin = FeatureFactoryUtils.lonToSphericalMercator(rectangle.getMinX()); + final double yMin = FeatureFactoryUtils.latToSphericalMercator(rectangle.getMinY()); + final double xMax = FeatureFactoryUtils.lonToSphericalMercator(rectangle.getMaxX()); + final double yMax = FeatureFactoryUtils.latToSphericalMercator(rectangle.getMaxY()); + final Coordinate[] coordinates = new Coordinate[5]; + coordinates[0] = new Coordinate(xMin, yMin); + coordinates[1] = new Coordinate(xMax, yMin); + coordinates[2] = new Coordinate(xMax, yMax); + coordinates[3] = new Coordinate(xMin, yMax); + coordinates[4] = new Coordinate(xMin, yMin); + return geomFactory.createPolygon(coordinates); + } + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java new file mode 100644 index 0000000000000..ced22f16f6d51 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.feature; + +import org.elasticsearch.geometry.Rectangle; +import org.locationtech.jts.geom.Envelope; + +/** + * Utility functions to transforms WGS84 coordinates into spherical mercator. + */ +class FeatureFactoryUtils { + + private FeatureFactoryUtils() { + // no instances + } + + /** + * Gets the JTS envelope for z/x/y/ tile in spherical mercator projection. + */ + public static Envelope getJTSTileBounds(int z, int x, int y) { + return new Envelope(getLong(x, z), getLong(x + 1, z), getLat(y, z), getLat(y + 1, z)); + } + + /** + * Gets the {@link org.elasticsearch.geometry.Geometry} envelope for z/x/y/ tile + * in spherical mercator projection. + */ + public static Rectangle getTileBounds(int z, int x, int y) { + return new Rectangle(getLong(x, z), getLong(x + 1, z), getLat(y, z), getLat(y + 1, z)); + } + + private static double getLong(int x, int zoom) { + return lonToSphericalMercator(Math.scalb(x, -zoom) * 360 - 180); + } + + private static double getLat(int y, int zoom) { + double r2d = 180 / Math.PI; + double n = Math.PI - 2 * Math.PI * y / Math.pow(2, zoom); + return latToSphericalMercator(r2d * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n)))); + } + + private static double MERCATOR_FACTOR = 20037508.34 / 180.0; + + /** + * Transforms WGS84 longitude to a Spherical mercator longitude + */ + public static double lonToSphericalMercator(double lon) { + return lon * MERCATOR_FACTOR; + } + + /** + * Transforms WGS84 latitude to a Spherical mercator latitude + */ + public static double latToSphericalMercator(double lat) { + double y = Math.log(Math.tan((90 + lat) * Math.PI / 360)) / (Math.PI / 180); + return y * MERCATOR_FACTOR; + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java new file mode 100644 index 0000000000000..53abad9ec1435 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java @@ -0,0 +1,72 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.feature; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.encoding.GeomCmd; +import com.wdtinc.mapbox_vector_tile.encoding.GeomCmdHdr; + +import org.apache.lucene.util.BitUtil; +import org.elasticsearch.geometry.Rectangle; + +/** + * Similar to {@link FeatureFactory} but only supports points and rectangles. It is just + * much more efficient for those shapes. + */ +public class SimpleFeatureFactory { + + private final int extent; + private final double pointXScale, pointYScale, pointXTranslate, pointYTranslate; + + public SimpleFeatureFactory(int z, int x, int y, int extent) { + this.extent = extent; + final Rectangle rectangle = FeatureFactoryUtils.getTileBounds(z, x, y); + pointXScale = (double) extent / (rectangle.getMaxLon() - rectangle.getMinLon()); + pointYScale = (double) -extent / (rectangle.getMaxLat() - rectangle.getMinLat()); + pointXTranslate = -pointXScale * rectangle.getMinX(); + pointYTranslate = -pointYScale * rectangle.getMinY(); + } + + public void point(VectorTile.Tile.Feature.Builder featureBuilder, double lon, double lat) { + featureBuilder.setType(VectorTile.Tile.GeomType.POINT); + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.MoveTo, 1)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(lon(lon))); + featureBuilder.addGeometry(BitUtil.zigZagEncode(lat(lat))); + } + + public void box(VectorTile.Tile.Feature.Builder featureBuilder, double minLon, double maxLon, double minLat, double maxLat) { + featureBuilder.setType(VectorTile.Tile.GeomType.POLYGON); + final int minX = lon(minLon); + final int minY = lat(minLat); + final int maxX = lon(maxLon); + final int maxY = lat(maxLat); + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.MoveTo, 1)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(minX)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(minY)); + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.LineTo, 3)); + // 1 + featureBuilder.addGeometry(BitUtil.zigZagEncode(maxX - minX)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); + // 2 + featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(maxY - minY)); + // 3 + featureBuilder.addGeometry(BitUtil.zigZagEncode(minX - maxX)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); + // close + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.ClosePath, 1)); + } + + private int lat(double lat) { + return (int) Math.round(pointYScale * FeatureFactoryUtils.latToSphericalMercator(lat) + pointYTranslate) + extent; + } + + private int lon(double lon) { + return (int) Math.round(pointXScale * FeatureFactoryUtils.lonToSphericalMercator(lon) + pointXTranslate); + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java new file mode 100644 index 0000000000000..a7c4d5a13adb6 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java @@ -0,0 +1,301 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.vectortile.rest; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStream; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestResponseListener; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; +import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; +import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.elasticsearch.search.profile.SearchProfileShardResults; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.xpack.vectortile.feature.FeatureFactory; +import org.elasticsearch.xpack.vectortile.feature.SimpleFeatureFactory; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +/** + * Main class handling a call to the _mvt API. + */ +public class RestVectorTileAction extends BaseRestHandler { + + private static final String META_LAYER = "meta"; + private static final String HITS_LAYER = "hits"; + private static final String AGGS_LAYER = "aggs"; + + private static final String GRID_FIELD = "grid"; + private static final String BOUNDS_FIELD = "bounds"; + + private static final String COUNT_TAG = "_count"; + private static final String ID_TAG = "_id"; + + // mime type as defined by the mapbox vector tile specification + private static final String MIME_TYPE = "application/vnd.mapbox-vector-tile"; + + public RestVectorTileAction() {} + + @Override + public List routes() { + return org.elasticsearch.core.List.of(new Route(GET, "{index}/_mvt/{field}/{z}/{x}/{y}")); + } + + @Override + public String getName() { + return "vector_tile_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + // This will allow to cancel the search request if the http channel is closed + final RestCancellableNodeClient cancellableNodeClient = new RestCancellableNodeClient(client, restRequest.getHttpChannel()); + final VectorTileRequest request = VectorTileRequest.parseRestRequest(restRequest); + final SearchRequestBuilder searchRequestBuilder = searchRequestBuilder(cancellableNodeClient, request); + return channel -> searchRequestBuilder.execute(new RestResponseListener(channel) { + + @Override + public RestResponse buildResponse(SearchResponse searchResponse) throws Exception { + try (BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput())) { + // Even if there is no hits, we return a tile with the meta layer + final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); + ensureOpen(); + final SearchHit[] hits = searchResponse.getHits().getHits(); + if (hits.length > 0) { + tileBuilder.addLayers(buildHitsLayer(hits, request)); + } + ensureOpen(); + final SimpleFeatureFactory geomBuilder = new SimpleFeatureFactory( + request.getZ(), + request.getX(), + request.getY(), + request.getExtent() + ); + final InternalGeoTileGrid grid = searchResponse.getAggregations() != null + ? searchResponse.getAggregations().get(GRID_FIELD) + : null; + // TODO: should we expose the total number of buckets on InternalGeoTileGrid? + if (grid != null && grid.getBuckets().size() > 0) { + tileBuilder.addLayers(buildAggsLayer(grid, request, geomBuilder)); + } + ensureOpen(); + final InternalGeoBounds bounds = searchResponse.getAggregations() != null + ? searchResponse.getAggregations().get(BOUNDS_FIELD) + : null; + final Aggregations aggsWithoutGridAndBounds = searchResponse.getAggregations() == null + ? null + : new Aggregations( + searchResponse.getAggregations() + .asList() + .stream() + .filter(a -> GRID_FIELD.equals(a.getName()) == false && BOUNDS_FIELD.equals(a.getName()) == false) + .collect(Collectors.toList()) + ); + final SearchResponse meta = new SearchResponse( + new SearchResponseSections( + new SearchHits( + SearchHits.EMPTY, + searchResponse.getHits().getTotalHits(), + searchResponse.getHits().getMaxScore() + ), // remove actual hits + aggsWithoutGridAndBounds, + searchResponse.getSuggest(), + searchResponse.isTimedOut(), + searchResponse.isTerminatedEarly(), + searchResponse.getProfileResults() == null + ? null + : new SearchProfileShardResults(searchResponse.getProfileResults()), + searchResponse.getNumReducePhases() + ), + searchResponse.getScrollId(), + searchResponse.getTotalShards(), + searchResponse.getSuccessfulShards(), + searchResponse.getSkippedShards(), + searchResponse.getTook().millis(), + searchResponse.getShardFailures(), + searchResponse.getClusters() + ); + tileBuilder.addLayers(buildMetaLayer(meta, bounds, request, geomBuilder)); + ensureOpen(); + tileBuilder.build().writeTo(bytesOut); + return new BytesRestResponse(RestStatus.OK, MIME_TYPE, bytesOut.bytes()); + } + } + }); + } + + private static SearchRequestBuilder searchRequestBuilder(RestCancellableNodeClient client, VectorTileRequest request) + throws IOException { + final SearchRequestBuilder searchRequestBuilder = client.prepareSearch(request.getIndexes()); + searchRequestBuilder.setSize(request.getSize()); + searchRequestBuilder.setFetchSource(false); + // TODO: I wonder if we can leverage field and format so what we get in the result is already the mvt commands. + searchRequestBuilder.addFetchField(new FieldAndFormat(request.getField(), null)); + for (FieldAndFormat field : request.getFieldAndFormats()) { + searchRequestBuilder.addFetchField(field); + } + searchRequestBuilder.setRuntimeMappings(request.getRuntimeMappings()); + QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(request.getField(), request.getBoundingBox()); + if (request.getQueryBuilder() != null) { + final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.filter(request.getQueryBuilder()); + boolQueryBuilder.filter(qBuilder); + qBuilder = boolQueryBuilder; + } + searchRequestBuilder.setQuery(qBuilder); + if (request.getGridPrecision() > 0) { + final Rectangle rectangle = request.getBoundingBox(); + final GeoBoundingBox boundingBox = new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + final int extent = 1 << request.getGridPrecision(); + final GeoGridAggregationBuilder tileAggBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) + .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getGridPrecision())) + .setGeoBoundingBox(boundingBox) + .size(extent * extent); + searchRequestBuilder.addAggregation(tileAggBuilder); + searchRequestBuilder.addAggregation(new StatsBucketPipelineAggregationBuilder(COUNT_TAG, GRID_FIELD + "." + COUNT_TAG)); + final AggregatorFactories.Builder otherAggBuilder = request.getAggBuilder(); + if (otherAggBuilder != null) { + tileAggBuilder.subAggregations(request.getAggBuilder()); + final Collection aggregations = otherAggBuilder.getAggregatorFactories(); + for (AggregationBuilder aggregation : aggregations) { + searchRequestBuilder.addAggregation( + new StatsBucketPipelineAggregationBuilder(aggregation.getName(), GRID_FIELD + ">" + aggregation.getName()) + ); + } + } + } + if (request.getExactBounds()) { + final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) + .wrapLongitude(false); + searchRequestBuilder.addAggregation(boundsBuilder); + } + for (SortBuilder sortBuilder : request.getSortBuilders()) { + searchRequestBuilder.addSort(sortBuilder); + } + return searchRequestBuilder; + } + + private static VectorTile.Tile.Layer.Builder buildHitsLayer(SearchHit[] hits, VectorTileRequest request) { + final FeatureFactory featureFactory = new FeatureFactory(request.getZ(), request.getX(), request.getY(), request.getExtent()); + final GeometryParser parser = new GeometryParser(true, false, false); + final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, request.getExtent()); + final List fields = request.getFieldAndFormats(); + for (SearchHit searchHit : hits) { + final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> { + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId()); + if (fields != null) { + for (FieldAndFormat field : fields) { + final DocumentField documentField = searchHit.field(field.field); + if (documentField != null) { + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue()); + } + } + } + }; + // TODO: See comment on field formats. + final Geometry geometry = parser.parseGeometry(searchHit.field(request.getField()).getValue()); + hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags)); + } + VectorTileUtils.addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); + return hitsLayerBuilder; + } + + private static VectorTile.Tile.Layer.Builder buildAggsLayer( + InternalGeoTileGrid grid, + VectorTileRequest request, + SimpleFeatureFactory geomBuilder + ) throws IOException { + final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + for (InternalGeoGridBucket bucket : grid.getBuckets()) { + featureBuilder.clear(); + // Add geometry + if (request.getGridType() == VectorTileRequest.GRID_TYPE.GRID) { + final Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); + geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + } else { + // TODO: it should be the centroid of the data? + final GeoPoint point = (GeoPoint) bucket.getKey(); + geomBuilder.point(featureBuilder, point.lon(), point.lat()); + } + // Add count as key value pair + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, bucket.getDocCount()); + for (Aggregation aggregation : bucket.getAggregations()) { + VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, aggregation); + } + aggLayerBuilder.addFeatures(featureBuilder); + } + VectorTileUtils.addPropertiesToLayer(aggLayerBuilder, layerProps); + return aggLayerBuilder; + } + + private static VectorTile.Tile.Layer.Builder buildMetaLayer( + SearchResponse response, + InternalGeoBounds bounds, + VectorTileRequest request, + SimpleFeatureFactory geomBuilder + ) throws IOException { + final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + if (bounds != null && bounds.topLeft() != null) { + final GeoPoint topLeft = bounds.topLeft(); + final GeoPoint bottomRight = bounds.bottomRight(); + geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); + } else { + final Rectangle tile = request.getBoundingBox(); + geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); + } + VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, response); + metaLayerBuilder.addFeatures(featureBuilder); + VectorTileUtils.addPropertiesToLayer(metaLayerBuilder, layerProps); + return metaLayerBuilder; + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java new file mode 100644 index 0000000000000..34a136aba21f4 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java @@ -0,0 +1,309 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.rest; + +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ParseField; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.CheckedFunction; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.elasticsearch.search.sort.SortBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + +/** + * Transforms a rest request in a vector tile request + */ +class VectorTileRequest { + + protected static final String INDEX_PARAM = "index"; + protected static final String FIELD_PARAM = "field"; + protected static final String Z_PARAM = "z"; + protected static final String X_PARAM = "x"; + protected static final String Y_PARAM = "y"; + + protected static final ParseField GRID_PRECISION_FIELD = new ParseField("grid_precision"); + protected static final ParseField GRID_TYPE_FIELD = new ParseField("grid_type"); + protected static final ParseField EXTENT_FIELD = new ParseField("extent"); + protected static final ParseField EXACT_BOUNDS_FIELD = new ParseField("exact_bounds"); + + protected enum GRID_TYPE { + GRID, + POINT; + + private static GRID_TYPE fromString(String type) { + switch (type.toLowerCase(Locale.ROOT)) { + case "grid": + return GRID; + case "point": + return POINT; + default: + throw new IllegalArgumentException("Invalid grid type [" + type + "]"); + } + } + } + + protected static class Defaults { + // TODO: Should it be SearchService.DEFAULT_SIZE? + public static final int SIZE = 10000; + public static final List FETCH = emptyList(); + public static final Map RUNTIME_MAPPINGS = emptyMap(); + public static final QueryBuilder QUERY = null; + public static final AggregatorFactories.Builder AGGS = null; + public static final List> SORT = emptyList(); + // TODO: Should it be 0, no aggs by default? + public static final int GRID_PRECISION = 8; + public static final GRID_TYPE GRID_TYPE = VectorTileRequest.GRID_TYPE.GRID; + public static final int EXTENT = 4096; + public static final boolean EXACT_BOUNDS = false; + } + + private static final ObjectParser PARSER; + + static { + PARSER = new ObjectParser<>("vector-tile"); + PARSER.declareInt(VectorTileRequest::setSize, SearchSourceBuilder.SIZE_FIELD); + PARSER.declareField(VectorTileRequest::setFieldAndFormats, (p) -> { + List fetchFields = new ArrayList<>(); + while ((p.nextToken()) != XContentParser.Token.END_ARRAY) { + fetchFields.add(FieldAndFormat.fromXContent(p)); + } + return fetchFields; + }, SearchSourceBuilder.FETCH_FIELDS_FIELD, ObjectParser.ValueType.OBJECT_ARRAY); + PARSER.declareField( + VectorTileRequest::setQueryBuilder, + (CheckedFunction) AbstractQueryBuilder::parseInnerQueryBuilder, + SearchSourceBuilder.QUERY_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField( + VectorTileRequest::setRuntimeMappings, + XContentParser::map, + SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField( + VectorTileRequest::setAggBuilder, + AggregatorFactories::parseAggregators, + SearchSourceBuilder.AGGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField( + VectorTileRequest::setSortBuilders, + SortBuilder::fromXContent, + SearchSourceBuilder.SORT_FIELD, + ObjectParser.ValueType.OBJECT_ARRAY + ); + // Specific for vector tiles + PARSER.declareInt(VectorTileRequest::setGridPrecision, GRID_PRECISION_FIELD); + PARSER.declareString(VectorTileRequest::setGridType, GRID_TYPE_FIELD); + PARSER.declareInt(VectorTileRequest::setExtent, EXTENT_FIELD); + PARSER.declareBoolean(VectorTileRequest::setExactBounds, EXACT_BOUNDS_FIELD); + } + + static VectorTileRequest parseRestRequest(RestRequest restRequest) throws IOException { + final VectorTileRequest request = new VectorTileRequest( + Strings.splitStringByCommaToArray(restRequest.param(INDEX_PARAM)), + restRequest.param(FIELD_PARAM), + Integer.parseInt(restRequest.param(Z_PARAM)), + Integer.parseInt(restRequest.param(X_PARAM)), + Integer.parseInt(restRequest.param(Y_PARAM)) + ); + if (restRequest.hasContent()) { + try (XContentParser contentParser = restRequest.contentParser()) { + PARSER.parse(contentParser, request, restRequest); + } + } + return request; + } + + private final String[] indexes; + private final String field; + private final int x; + private final int y; + private final int z; + private final Rectangle bbox; + private QueryBuilder queryBuilder = Defaults.QUERY; + private Map runtimeMappings = Defaults.RUNTIME_MAPPINGS; + private int gridPrecision = Defaults.GRID_PRECISION; + private GRID_TYPE gridType = Defaults.GRID_TYPE; + private int size = Defaults.SIZE; + private int extent = Defaults.EXTENT; + private AggregatorFactories.Builder aggBuilder = Defaults.AGGS; + private List fields = Defaults.FETCH; + private List> sortBuilders = Defaults.SORT; + private boolean exact_bounds = Defaults.EXACT_BOUNDS; + + private VectorTileRequest(String[] indexes, String field, int z, int x, int y) { + this.indexes = indexes; + this.field = field; + this.z = z; + this.x = x; + this.y = y; + // This should validate that z/x/y is a valid combination + this.bbox = GeoTileUtils.toBoundingBox(x, y, z); + } + + public String[] getIndexes() { + return indexes; + } + + public String getField() { + return field; + } + + public int getX() { + return x; + } + + public int getY() { + return y; + } + + public int getZ() { + return z; + } + + public Rectangle getBoundingBox() { + return bbox; + } + + public int getExtent() { + return extent; + } + + private void setExtent(int extent) { + if (extent < 0) { + throw new IllegalArgumentException("[extent] parameter cannot be negative, found [" + extent + "]"); + } + this.extent = extent; + } + + public boolean getExactBounds() { + return exact_bounds; + } + + private void setExactBounds(boolean exact_bounds) { + this.exact_bounds = exact_bounds; + } + + public List getFieldAndFormats() { + return fields; + } + + private void setFieldAndFormats(List fields) { + this.fields = fields; + } + + public QueryBuilder getQueryBuilder() { + return queryBuilder; + } + + private void setQueryBuilder(QueryBuilder queryBuilder) { + // TODO: validation + this.queryBuilder = queryBuilder; + } + + public Map getRuntimeMappings() { + return runtimeMappings; + } + + private void setRuntimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + } + + public int getGridPrecision() { + return gridPrecision; + } + + private void setGridPrecision(int gridPrecision) { + if (gridPrecision < 0) { + throw new IllegalArgumentException("[gridPrecision] parameter cannot be negative, found [" + gridPrecision + "]"); + } + if (gridPrecision > 8) { + throw new IllegalArgumentException("[gridPrecision] parameter cannot be bigger than 8, found [" + gridPrecision + "]"); + } + this.gridPrecision = gridPrecision; + } + + public GRID_TYPE getGridType() { + return gridType; + } + + private void setGridType(String gridType) { + this.gridType = GRID_TYPE.fromString(gridType); + } + + public int getSize() { + return size; + } + + private void setSize(int size) { + if (size < 0) { + throw new IllegalArgumentException("[size] parameter cannot be negative, found [" + size + "]"); + } + this.size = size; + } + + public AggregatorFactories.Builder getAggBuilder() { + return aggBuilder; + } + + private void setAggBuilder(AggregatorFactories.Builder aggBuilder) { + for (AggregationBuilder aggregation : aggBuilder.getAggregatorFactories()) { + final String type = aggregation.getType(); + switch (type) { + case MinAggregationBuilder.NAME: + case MaxAggregationBuilder.NAME: + case AvgAggregationBuilder.NAME: + case SumAggregationBuilder.NAME: + case CardinalityAggregationBuilder.NAME: + break; + default: + // top term and percentile should be supported + throw new IllegalArgumentException("Unsupported aggregation of type [" + type + "]"); + } + } + for (PipelineAggregationBuilder aggregation : aggBuilder.getPipelineAggregatorFactories()) { + // should not have pipeline aggregations + final String type = aggregation.getType(); + throw new IllegalArgumentException("Unsupported pipeline aggregation of type [" + type + "]"); + } + this.aggBuilder = aggBuilder; + } + + public List> getSortBuilders() { + return sortBuilders; + } + + private void setSortBuilders(List> sortBuilders) { + this.sortBuilders = sortBuilders; + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java new file mode 100644 index 0000000000000..09fd711942965 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.rest; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; +import com.wdtinc.mapbox_vector_tile.encoding.MvtValue; + +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; + +import java.io.IOException; +import java.util.Map; + +/** + * Utility methods for vector tiles. + */ +class VectorTileUtils { + + private VectorTileUtils() { + // no instances + } + + /** + * Creates a vector layer builder with the provided name and extent. + */ + public static VectorTile.Tile.Layer.Builder createLayerBuilder(String layerName, int extent) { + final VectorTile.Tile.Layer.Builder layerBuilder = VectorTile.Tile.Layer.newBuilder(); + layerBuilder.setVersion(2); + layerBuilder.setName(layerName); + layerBuilder.setExtent(extent); + return layerBuilder; + } + + /** + * Adds the flatten elements of toXContent into the feature as tags. + */ + public static void addToXContentToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, ToXContent toXContent) + throws IOException { + final Map map = Maps.flatten( + XContentHelper.convertToMap(XContentHelper.toXContent(toXContent, XContentType.CBOR, false), true, XContentType.CBOR).v2(), + true, + true + ); + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() != null) { + addPropertyToFeature(feature, layerProps, entry.getKey(), entry.getValue()); + } + } + } + + /** + * Adds the provided key / value pair into the feature as tags. + */ + public static void addPropertyToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, String key, Object value) { + feature.addTags(layerProps.addKey(key)); + feature.addTags(layerProps.addValue(value)); + } + + /** + * Adds the given properties to the provided layer. + */ + public static void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, MvtLayerProps layerProps) { + // Add keys + layer.addAllKeys(layerProps.getKeys()); + // Add values + final Iterable values = layerProps.getVals(); + for (Object value : values) { + layer.addValues(MvtValue.toValue(value)); + } + } +} diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java new file mode 100644 index 0000000000000..7076bb8f6ec6d --- /dev/null +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.feature; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; + +import org.apache.lucene.geo.GeoTestUtil; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.util.Arrays; +import java.util.List; + +public class FeatureFactoryTests extends ESTestCase { + + public void testPoint() { + int z = randomIntBetween(1, 10); + int x = randomIntBetween(0, (1 << z) - 1); + int y = randomIntBetween(0, (1 << z) - 1); + int extent = randomIntBetween(1 << 8, 1 << 14); + // check if we might have numerical error due to floating point arithmetic + assumeFalse("", hasNumericalError(z, x, y, extent)); + Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); + SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); + FeatureFactory factory = new FeatureFactory(z, x, y, extent); + VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + for (int i = 0; i < 10; i++) { + featureBuilder.clear(); + double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() > l || rectangle.getMaxY() < l, GeoTestUtil::nextLatitude); + double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() > l || rectangle.getMaxX() < l, GeoTestUtil::nextLongitude); + builder.point(featureBuilder, lon, lat); + byte[] b1 = featureBuilder.build().toByteArray(); + Point point = new Point(lon, lat); + List features = factory.getFeatures(point, new UserDataIgnoreConverter()); + assertThat(features.size(), Matchers.equalTo(1)); + byte[] b2 = features.get(0).toByteArray(); + assertArrayEquals(b1, b2); + } + } + + public void testIssue74341() { + int z = 1; + int x = 0; + int y = 0; + int extent = 1730; + // this is the typical case we need to guard from. + assertThat(hasNumericalError(z, x, y, extent), Matchers.equalTo(true)); + double lon = -171.0; + double lat = 0.9999999403953552; + SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); + FeatureFactory factory = new FeatureFactory(z, x, y, extent); + VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + builder.point(featureBuilder, lon, lat); + byte[] b1 = featureBuilder.build().toByteArray(); + Point point = new Point(lon, lat); + List features = factory.getFeatures(point, new UserDataIgnoreConverter()); + assertThat(features.size(), Matchers.equalTo(1)); + byte[] b2 = features.get(0).toByteArray(); + assertThat(Arrays.equals(b1, b2), Matchers.equalTo(false)); + } + + private boolean hasNumericalError(int z, int x, int y, int extent) { + final Rectangle rectangle = FeatureFactoryUtils.getTileBounds(z, x, y); + final double xDiff = rectangle.getMaxLon() - rectangle.getMinLon(); + final double yDiff = rectangle.getMaxLat() - rectangle.getMinLat(); + return (double) -extent / yDiff != -1d / (yDiff / (double) extent) || (double) extent / xDiff != 1d / (xDiff / (double) extent); + } + + public void testRectangle() { + int z = randomIntBetween(1, 10); + int x = randomIntBetween(0, (1 << z) - 1); + int y = randomIntBetween(0, (1 << z) - 1); + int extent = randomIntBetween(1 << 8, 1 << 14); + SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); + FeatureFactory factory = new FeatureFactory(z, x, y, extent); + Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + for (int i = 0; i < extent; i++) { + featureBuilder.clear(); + builder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + byte[] b1 = featureBuilder.build().toByteArray(); + List features = factory.getFeatures(r, new UserDataIgnoreConverter()); + assertThat(features.size(), Matchers.equalTo(1)); + byte[] b2 = features.get(0).toByteArray(); + assertArrayEquals(extent + "", b1, b2); + } + } +} diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java new file mode 100644 index 0000000000000..f168acdead0ee --- /dev/null +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java @@ -0,0 +1,280 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.rest; + +import com.carrotsearch.randomizedtesting.generators.RandomPicks; + +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.core.CheckedConsumer; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.function.Consumer; + +import static java.util.Collections.emptyList; + +public class VectorTileRequestTests extends ESTestCase { + + @Override + protected NamedXContentRegistry xContentRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, false, emptyList()); + return new NamedXContentRegistry(searchModule.getNamedXContents()); + } + + public void testDefaults() throws IOException { + assertRestRequest((builder) -> {}, (vectorTileRequest) -> { + assertThat(vectorTileRequest.getSize(), Matchers.equalTo(VectorTileRequest.Defaults.SIZE)); + assertThat(vectorTileRequest.getExtent(), Matchers.equalTo(VectorTileRequest.Defaults.EXTENT)); + assertThat(vectorTileRequest.getAggBuilder(), Matchers.equalTo(VectorTileRequest.Defaults.AGGS)); + assertThat(vectorTileRequest.getFieldAndFormats(), Matchers.equalTo(VectorTileRequest.Defaults.FETCH)); + assertThat(vectorTileRequest.getGridType(), Matchers.equalTo(VectorTileRequest.Defaults.GRID_TYPE)); + assertThat(vectorTileRequest.getGridPrecision(), Matchers.equalTo(VectorTileRequest.Defaults.GRID_PRECISION)); + assertThat(vectorTileRequest.getExactBounds(), Matchers.equalTo(VectorTileRequest.Defaults.EXACT_BOUNDS)); + assertThat(vectorTileRequest.getRuntimeMappings(), Matchers.equalTo(VectorTileRequest.Defaults.RUNTIME_MAPPINGS)); + assertThat(vectorTileRequest.getSortBuilders(), Matchers.equalTo(VectorTileRequest.Defaults.SORT)); + assertThat(vectorTileRequest.getQueryBuilder(), Matchers.equalTo(VectorTileRequest.Defaults.QUERY)); + }); + } + + public void testFieldSize() throws IOException { + final int size = randomIntBetween(0, 10000); + assertRestRequest( + (builder) -> { builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getSize(), Matchers.equalTo(size)); } + ); + } + + public void testFieldExtent() throws IOException { + final int extent = randomIntBetween(256, 8192); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.EXTENT_FIELD.getPreferredName(), extent); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getExtent(), Matchers.equalTo(extent)); } + ); + } + + public void testFieldFetch() throws IOException { + final String fetchField = randomAlphaOfLength(10); + assertRestRequest( + (builder) -> { builder.field(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName(), new String[] { fetchField }); }, + (vectorTileRequest) -> { + assertThat(vectorTileRequest.getFieldAndFormats(), Matchers.iterableWithSize(1)); + assertThat(vectorTileRequest.getFieldAndFormats().get(0).field, Matchers.equalTo(fetchField)); + } + ); + } + + public void testFieldGridType() throws IOException { + final VectorTileRequest.GRID_TYPE grid_type = RandomPicks.randomFrom(random(), VectorTileRequest.GRID_TYPE.values()); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.GRID_TYPE_FIELD.getPreferredName(), grid_type.name()); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getGridType(), Matchers.equalTo(grid_type)); } + ); + } + + public void testFieldGridPrecision() throws IOException { + final int grid_precision = randomIntBetween(1, 8); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.GRID_PRECISION_FIELD.getPreferredName(), grid_precision); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getGridPrecision(), Matchers.equalTo(grid_precision)); } + ); + } + + public void testFieldExactBounds() throws IOException { + final boolean exactBounds = randomBoolean(); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.EXACT_BOUNDS_FIELD.getPreferredName(), exactBounds); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getExactBounds(), Matchers.equalTo(exactBounds)); } + ); + } + + public void testFieldQuery() throws IOException { + final QueryBuilder queryBuilder = new TermQueryBuilder(randomAlphaOfLength(10), randomAlphaOfLength(10)); + assertRestRequest((builder) -> { + builder.field(SearchSourceBuilder.QUERY_FIELD.getPreferredName()); + queryBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); + }, (vectorTileRequest) -> { assertThat(vectorTileRequest.getQueryBuilder(), Matchers.equalTo(queryBuilder)); }); + } + + public void testFieldAgg() throws IOException { + final AggregationBuilder aggregationBuilder = new AvgAggregationBuilder("xxx").field("xxxx"); + assertRestRequest((builder) -> { + builder.startObject(SearchSourceBuilder.AGGS_FIELD.getPreferredName()); + aggregationBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + }, (vectorTileRequest) -> { + assertThat(vectorTileRequest.getAggBuilder().getAggregatorFactories(), Matchers.iterableWithSize(1)); + assertThat(vectorTileRequest.getAggBuilder().getAggregatorFactories().contains(aggregationBuilder), Matchers.equalTo(true)); + }); + } + + public void testFieldRuntimeMappings() throws IOException { + final String fieldName = randomAlphaOfLength(10); + assertRestRequest((builder) -> { + builder.startObject(SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD.getPreferredName()) + .startObject(fieldName) + .field("script", "emit('foo')") + .field("type", "string") + .endObject() + .endObject(); + }, (vectorTileRequest) -> { + assertThat(vectorTileRequest.getRuntimeMappings(), Matchers.aMapWithSize(1)); + assertThat(vectorTileRequest.getRuntimeMappings().get(fieldName), Matchers.notNullValue()); + }); + } + + public void testFieldSort() throws IOException { + final String sortName = randomAlphaOfLength(10); + assertRestRequest( + (builder) -> { + builder.startArray(SearchSourceBuilder.SORT_FIELD.getPreferredName()) + .startObject() + .field(sortName, "desc") + .endObject() + .endArray(); + }, + (vectorTileRequest) -> { + assertThat(vectorTileRequest.getSortBuilders(), Matchers.iterableWithSize(1)); + FieldSortBuilder sortBuilder = (FieldSortBuilder) vectorTileRequest.getSortBuilders().get(0); + assertThat(sortBuilder.getFieldName(), Matchers.equalTo(sortName)); + } + ); + } + + public void testWrongTile() { + final String index = randomAlphaOfLength(10); + final String field = randomAlphaOfLength(10); + { + // negative zoom + final int z = randomIntBetween(Integer.MIN_VALUE, -1); + final int x = 0; + final int y = 0; + final FakeRestRequest request = getBasicRequestBuilder(index, field, z, x, y).build(); + final IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> VectorTileRequest.parseRestRequest(request) + ); + assertThat(ex.getMessage(), Matchers.equalTo("Invalid geotile_grid precision of " + z + ". Must be between 0 and 29.")); + } + { + // too big zoom + final int z = -randomIntBetween(GeoTileUtils.MAX_ZOOM + 1, Integer.MAX_VALUE); + final int x = 0; + final int y = 0; + final FakeRestRequest request = getBasicRequestBuilder(index, field, z, x, y).build(); + final IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> VectorTileRequest.parseRestRequest(request) + ); + assertThat(ex.getMessage(), Matchers.equalTo("Invalid geotile_grid precision of " + z + ". Must be between 0 and 29.")); + } + { + // negative x + final int z = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + final int x = randomIntBetween(Integer.MIN_VALUE, -1); + final int y = randomIntBetween(0, (1 << z) - 1); + final FakeRestRequest request = getBasicRequestBuilder(index, field, z, x, y).build(); + final IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> VectorTileRequest.parseRestRequest(request) + ); + assertThat(ex.getMessage(), Matchers.equalTo("Zoom/X/Y combination is not valid: " + z + "/" + x + "/" + y)); + } + { + // too big x + final int z = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + final int x = randomIntBetween(Integer.MIN_VALUE, -1); + final int y = randomIntBetween(1 << z, Integer.MAX_VALUE); + final FakeRestRequest request = getBasicRequestBuilder(index, field, z, x, y).build(); + final IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> VectorTileRequest.parseRestRequest(request) + ); + assertThat(ex.getMessage(), Matchers.equalTo("Zoom/X/Y combination is not valid: " + z + "/" + x + "/" + y)); + } + { + // negative y + final int z = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + final int x = randomIntBetween(0, (1 << z) - 1); + final int y = randomIntBetween(Integer.MIN_VALUE, -1); + final FakeRestRequest request = getBasicRequestBuilder(index, field, z, x, y).build(); + final IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> VectorTileRequest.parseRestRequest(request) + ); + assertThat(ex.getMessage(), Matchers.equalTo("Zoom/X/Y combination is not valid: " + z + "/" + x + "/" + y)); + } + { + // too big y + final int z = randomIntBetween(0, GeoTileUtils.MAX_ZOOM); + final int x = randomIntBetween(1 << z, Integer.MAX_VALUE); + final int y = randomIntBetween(Integer.MIN_VALUE, -1); + final FakeRestRequest request = getBasicRequestBuilder(index, field, z, x, y).build(); + final IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> VectorTileRequest.parseRestRequest(request) + ); + assertThat(ex.getMessage(), Matchers.equalTo("Zoom/X/Y combination is not valid: " + z + "/" + x + "/" + y)); + } + } + + private void assertRestRequest(CheckedConsumer consumer, Consumer asserter) + throws IOException { + final int z = randomIntBetween(1, 10); + final int x = randomIntBetween(0, (1 << z) - 1); + final int y = randomIntBetween(0, (1 << z) - 1); + final String index = randomAlphaOfLength(10); + final String field = randomAlphaOfLength(10); + final FakeRestRequest.Builder requestBuilder = getBasicRequestBuilder(index, field, z, x, y); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + consumer.accept(builder); + builder.endObject(); + final FakeRestRequest request = requestBuilder.withContent(BytesReference.bytes(builder), builder.contentType()).build(); + final VectorTileRequest vectorTileRequest = VectorTileRequest.parseRestRequest(request); + assertThat(vectorTileRequest.getIndexes(), Matchers.equalTo(new String[] { index })); + assertThat(vectorTileRequest.getField(), Matchers.equalTo(field)); + assertThat(vectorTileRequest.getZ(), Matchers.equalTo(z)); + assertThat(vectorTileRequest.getX(), Matchers.equalTo(x)); + assertThat(vectorTileRequest.getY(), Matchers.equalTo(y)); + asserter.accept(vectorTileRequest); + } + + private FakeRestRequest.Builder getBasicRequestBuilder(String index, String field, int z, int x, int y) { + return new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withParams( + org.elasticsearch.core.Map.of( + VectorTileRequest.INDEX_PARAM, + index, + VectorTileRequest.FIELD_PARAM, + field, + VectorTileRequest.Z_PARAM, + "" + z, + VectorTileRequest.X_PARAM, + "" + x, + VectorTileRequest.Y_PARAM, + "" + y + ) + ); + } +}