Merge feature/vector tiles branch to master

server/src/main/java/org/elasticsearch/common/util/Maps.java

@@ -11,8 +11,10 @@
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
+import java.util.TreeMap;
 import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
@@ -62,7 +64,7 @@ public static <K, V> Map<K, V> copyMapWithAddedOrReplacedEntry(final Map<K, V> m
         Objects.requireNonNull(value);
         assert checkIsImmutableMap(map, key, value);
         return Stream.concat(map.entrySet().stream().filter(k -> key.equals(k.getKey()) == false), Stream.of(entry(key, value)))
-                .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
+            .collect(Collectors.toUnmodifiableMap(Map.Entry::getKey, Map.Entry::getValue));
     }
 
     /**
@@ -85,10 +87,10 @@ public static <K, V> Map<K, V> copyMapWithRemovedEntry(final Map<K, V> map, fina
 
     // map classes that are known to be immutable, used to speed up immutability check in #assertImmutableMap
     private static final Set<Class<?>> IMMUTABLE_MAP_CLASSES = Set.of(
-            Collections.emptyMap().getClass(),
-            Collections.unmodifiableMap(new HashMap<>()).getClass(),
-            Map.of().getClass(),
-            Map.of("a", "b").getClass()
+        Collections.emptyMap().getClass(),
+        Collections.unmodifiableMap(new HashMap<>()).getClass(),
+        Map.of().getClass(),
+        Map.of("a", "b").getClass()
     );
 
     private static <K, V> boolean checkIsImmutableMap(final Map<K, V> map, final K key, final V value) {
@@ -135,7 +137,56 @@ public static <K, V> boolean deepEquals(Map<K, V> left, Map<K, V> right) {
             return false;
         }
         return left.entrySet().stream()
-                .allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey())));
+            .allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey())));
+    }
+
+    /**
+     * Returns an array where all internal maps and optionally arrays are flattened into the root map.
+     *
+     * For example the map {"foo": {"bar": 1, "baz": [2, 3]}} will become {"foo.bar": 1, "foo.baz.0": 2, "foo.baz.1": 3}. Note that if
+     * maps contains keys with "." or numbers it is possible that such keys will be silently overridden. For example the map
+     * {"foo": {"bar": 1}, "foo.bar": 2} will become {"foo.bar": 1} or {"foo.bar": 2}.
+     *
+     * @param map - input to be flattened
+     * @param flattenArrays - if false, arrays will be ignored
+     * @param ordered - if true the resulted map will be sorted
+     * @return
+     */
+    public static Map<String, Object> flatten(Map<String, Object> map, boolean flattenArrays, boolean ordered) {
+        return flatten(map, flattenArrays, ordered, null);
     }
 
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> flatten(Map<String, Object> map, boolean flattenArrays, boolean ordered, String parentPath) {
+        Map<String, Object> flatMap = ordered ? new TreeMap<>() : new HashMap<>();
+        String prefix = parentPath != null ? parentPath + "." : "";
+        for (Map.Entry<String, Object> entry : map.entrySet()) {
+            if (entry.getValue() instanceof Map) {
+                flatMap.putAll(flatten((Map<String, Object>) entry.getValue(), flattenArrays, ordered, prefix + entry.getKey()));
+            } else if (flattenArrays && entry.getValue() instanceof List) {
+                flatMap.putAll(flatten((List<Object>) entry.getValue(), ordered, prefix + entry.getKey()));
+            } else {
+                flatMap.put(prefix + entry.getKey(), entry.getValue());
+            }
+        }
+        return flatMap;
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Map<String, Object> flatten(List<Object> list, boolean ordered, String parentPath) {
+        Map<String, Object> flatMap = ordered ? new TreeMap<>() : new HashMap<>();
+        String prefix = parentPath != null ? parentPath + "." : "";
+        for (int i = 0; i < list.size(); i++) {
+            Object cur = list.get(i);
+            if (cur instanceof Map) {
+                flatMap.putAll(flatten((Map<String, Object>) cur, true, ordered, prefix + i));
+            }
+            if (cur instanceof List) {
+                flatMap.putAll(flatten((List<Object>) cur, ordered, prefix + i));
+            } else {
+                flatMap.put(prefix + i, cur);
+            }
+        }
+        return flatMap;
+    }
 }

server/src/test/java/org/elasticsearch/common/util/MapsTests.java

@@ -15,6 +15,7 @@
 import java.util.Collection;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Map;
 import java.util.function.Supplier;
 import java.util.stream.Collectors;
@@ -122,6 +123,70 @@ public void testDeepEquals() {
         assertFalse(Maps.deepEquals(map, mapModified));
     }
 
+    public void testFlatten() {
+        Map<String, Object> map = randomNestedMap(10);
+        Map<String, Object> flatten = Maps.flatten(map, true, true);
+        assertThat(flatten.size(), equalTo(deepCount(map.values())));
+        for (Map.Entry<String, Object> entry : flatten.entrySet()) {
+            assertThat(entry.getKey(), entry.getValue(), equalTo(deepGet(entry.getKey(), map)));
+        }
+    }
+
+    @SuppressWarnings("unchecked")
+    private static Object deepGet(String path, Object obj) {
+        Object cur = obj;
+        String[] keys = path.split("\\.");
+        for (String key : keys) {
+            if (Character.isDigit(key.charAt(0))) {
+                List<Object> list = (List<Object>) cur;
+                cur = list.get(Integer.parseInt(key));
+             } else {
+                Map<String, Object> map = (Map<String, Object>) cur;
+                cur = map.get(key);
+            }
+        }
+        return cur;
+    }
+
+    @SuppressWarnings("unchecked")
+    private int deepCount(Collection<Object> map) {
+        int sum = 0;
+        for (Object val : map) {
+            if (val instanceof Map) {
+                sum += deepCount(((Map<String, Object>) val).values());
+            } else if (val instanceof List) {
+                sum += deepCount((List<Object>) val);
+            } else {
+                sum ++;
+            }
+        }
+        return sum;
+    }
+
+    private Map<String, Object> randomNestedMap(int level) {
+        final Supplier<String> keyGenerator = () -> randomAlphaOfLengthBetween(1, 5);
+        final Supplier<Object> arrayValueGenerator = () -> random().ints(randomInt(5))
+            .boxed()
+            .map(s -> (Object) s)
+            .collect(Collectors.toList());
+
+        final Supplier<Object> mapSupplier;
+        if (level > 0) {
+            mapSupplier = () -> randomNestedMap(level - 1);
+        } else {
+            mapSupplier = ESTestCase::randomLong;
+        }
+        final Supplier<Supplier<Object>> valueSupplier = () -> randomFrom(
+            ESTestCase::randomBoolean,
+            ESTestCase::randomDouble,
+            ESTestCase::randomLong,
+            arrayValueGenerator,
+            mapSupplier
+        );
+        final Supplier<Object> valueGenerator = () -> valueSupplier.get().get();
+        return randomMap(randomInt(5), keyGenerator, valueGenerator);
+    }
+
     private void assertMapEntries(final Map<String, String> map, final Collection<Map.Entry<String, String>> entries) {
         for (var entry : entries) {
             assertThat("map [" + map + "] does not contain key [" + entry.getKey() + "]", map.keySet(), hasItem(entry.getKey()));

x-pack/plugin/vector-tile/build.gradle

@@ -0,0 +1,69 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import org.elasticsearch.gradle.internal.info.BuildParams
+
+apply plugin: 'elasticsearch.internal-es-plugin'
+apply plugin: 'elasticsearch.java-rest-test'
+
+esplugin {
+  name 'vector-tile'
+  description 'A plugin for mapbox vector tile features'
+  classname 'org.elasticsearch.xpack.vectortile.VectorTilePlugin'
+  extendedPlugins = ['x-pack-core']
+}
+
+dependencies {
+  compileOnly project(path: xpackModule('core'))
+  testImplementation(testArtifact(project(xpackModule('core'))))
+  api "com.wdtinc:mapbox-vector-tile:3.1.0"
+  api "com.google.protobuf:protobuf-java:3.14.0"
+  javaRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0")
+  javaRestTestImplementation("com.google.protobuf:protobuf-java:3.14.0")
+}
+
+testClusters.all {
+  setting 'xpack.license.self_generated.type', 'trial'
+  testDistribution = 'DEFAULT'
+  setting 'xpack.security.enabled', 'false'
+  if (BuildParams.isSnapshotBuild() == false) {
+    systemProperty 'es.vector_tile_feature_flag_registered', 'true'
+  }
+}
+
+tasks.named("test").configure {
+  if (BuildParams.isSnapshotBuild() == false) {
+    systemProperty 'es.vector_tile_feature_flag_registered', 'true'
+  }
+}
+
+tasks.named("thirdPartyAudit").configure {
+  ignoreViolations(
+    // uses internal java api: sun.misc.Unsafe
+    'com.google.protobuf.UnsafeUtil',
+    'com.google.protobuf.MessageSchema',
+    'com.google.protobuf.UnsafeUtil$1',
+    'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor',
+    'com.google.protobuf.UnsafeUtil$MemoryAccessor'
+  )
+
+  ignoreMissingClasses(
+    'org.slf4j.Logger',
+    'org.slf4j.LoggerFactory'
+  )
+}
+

x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-3.1.0.jar.sha1

@@ -0,0 +1 @@
+06c4432c7885a3938571a57e73cc1444d7a39f12