From e026a17185b1a4c7d0e9751339ce52d1e67e3ea5 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Mon, 6 Jun 2022 13:41:56 -0700 Subject: [PATCH] Fix the support of RestClient Node Sniffer for version 2.x and update tests (#3487) Fix the support of RestClient Node Sniffer for OpenSearch 2.x, and update unit tests for OpenSearch. The current code contains the logic to be compatible with Elasticsearch 2.x version, which is conflict with OpenSearch 2.x, so removed that part of legacy code. * Update the script create_test_nodes_info.bash to dump the response of Nodes Info API GET _nodes/http for OpenSearch 1.0 and 2.0 version, which used for unit test. * Remove the support of Elasticsearch version 2.x for the Sniffer * Update unit test to validate the Sniffer compatible with OpenSearch 1.x and 2.x * Update the API response parser to meet the array notation (in ES 6.1 and above) for the node attributes setting. It will result the value of `node.attr` setting will not be parsed as array in the Sniffer, when using the Sniffer on cluster in Elasticsearch 6.0 and above. * Replace "master" node role with "cluster_manager" in unit test Signed-off-by: Tianli Feng (cherry picked from commit ab478ba5f33d45b98a3dd2b12038680d443cc1e1) --- .../client/sniff/OpenSearchNodesSniffer.java | 70 +----- .../OpenSearchNodesSnifferParseTests.java | 92 +++----- .../sniff/OpenSearchNodesSnifferTests.java | 16 +- ..._nodes_http.json => 1.0.0_nodes_http.json} | 79 ++++--- .../src/test/resources/2.0.0_nodes_http.json | 144 +++++++----- .../src/test/resources/5.0.0_nodes_http.json | 217 ------------------ .../src/test/resources/6.0.0_nodes_http.json | 217 ------------------ .../resources/create_test_nodes_info.bash | 56 +++-- 8 files changed, 191 insertions(+), 700 deletions(-) rename client/sniffer/src/test/resources/{7.3.0_nodes_http.json => 1.0.0_nodes_http.json} (77%) delete mode 100644 client/sniffer/src/test/resources/5.0.0_nodes_http.json delete mode 100644 client/sniffer/src/test/resources/6.0.0_nodes_http.json diff --git a/client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java b/client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java index 2829439627dbc..c1a0fcf9a8acf 100644 --- a/client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java +++ b/client/sniffer/src/main/java/org/opensearch/client/sniff/OpenSearchNodesSniffer.java @@ -49,6 +49,7 @@ import java.io.InputStream; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -241,74 +242,23 @@ private static Node readNode(String nodeId, JsonParser parser, Scheme scheme) th } Map> realAttributes = new HashMap<>(protoAttributes.size()); - List keys = new ArrayList<>(protoAttributes.keySet()); - for (String key : keys) { - if (key.endsWith(".0")) { - String realKey = key.substring(0, key.length() - 2); - List values = new ArrayList<>(); - int i = 0; - while (true) { - String value = protoAttributes.remove(realKey + "." + i); - if (value == null) { - break; - } - values.add(value); - i++; - } - realAttributes.put(realKey, unmodifiableList(values)); - } - } for (Map.Entry entry : protoAttributes.entrySet()) { - realAttributes.put(entry.getKey(), singletonList(entry.getValue())); - } - - if (version.startsWith("2.")) { - /* - * 2.x doesn't send roles, instead we try to read them from - * attributes. - */ - boolean clientAttribute = v2RoleAttributeValue(realAttributes, "client", false); - Boolean masterAttribute = v2RoleAttributeValue(realAttributes, "master", null); - Boolean dataAttribute = v2RoleAttributeValue(realAttributes, "data", null); - if ((masterAttribute == null && false == clientAttribute) || masterAttribute) { - roles.add("master"); + if (entry.getValue().startsWith("[")) { + // Convert string array to list + String value = entry.getValue(); + String[] values = value.substring(1, value.length() - 1).split(", "); + realAttributes.put(entry.getKey(), unmodifiableList(Arrays.asList(values))); + } else { + realAttributes.put(entry.getKey(), singletonList(entry.getValue())); } - if ((dataAttribute == null && false == clientAttribute) || dataAttribute) { - roles.add("data"); - } - } else { - assert sawRoles : "didn't see roles for [" + nodeId + "]"; } + + assert sawRoles : "didn't see roles for [" + nodeId + "]"; assert boundHosts.contains(publishedHost) : "[" + nodeId + "] doesn't make sense! publishedHost should be in boundHosts"; logger.trace("adding node [" + nodeId + "]"); return new Node(publishedHost, boundHosts, name, version, new Roles(roles), unmodifiableMap(realAttributes)); } - /** - * Returns {@code defaultValue} if the attribute didn't come back, - * {@code true} or {@code false} if it did come back as - * either of those, or throws an IOException if the attribute - * came back in a strange way. - */ - private static Boolean v2RoleAttributeValue(Map> attributes, String name, Boolean defaultValue) - throws IOException { - List valueList = attributes.remove(name); - if (valueList == null) { - return defaultValue; - } - if (valueList.size() != 1) { - throw new IOException("expected only a single attribute value for [" + name + "] but got " + valueList); - } - switch (valueList.get(0)) { - case "true": - return true; - case "false": - return false; - default: - throw new IOException("expected [" + name + "] to be either [true] or [false] but was [" + valueList.get(0) + "]"); - } - } - /** * The supported host schemes. */ diff --git a/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java b/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java index a9ff47eab5366..58b60ac13dee8 100644 --- a/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java +++ b/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferParseTests.java @@ -45,8 +45,8 @@ import java.io.IOException; import java.io.InputStream; import java.util.Arrays; -import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -85,59 +85,31 @@ private void checkFile(String file, Node... expected) throws IOException { } } - public void test2x() throws IOException { - checkFile( - "2.0.0_nodes_http.json", - node(9200, "m1", "2.0.0", true, false, false), - node(9201, "m2", "2.0.0", true, true, false), - node(9202, "m3", "2.0.0", true, false, false), - node(9203, "d1", "2.0.0", false, true, false), - node(9204, "d2", "2.0.0", false, true, false), - node(9205, "d3", "2.0.0", false, true, false), - node(9206, "c1", "2.0.0", false, false, false), - node(9207, "c2", "2.0.0", false, false, false) - ); - } - - public void test5x() throws IOException { - checkFile( - "5.0.0_nodes_http.json", - node(9200, "m1", "5.0.0", true, false, true), - node(9201, "m2", "5.0.0", true, true, true), - node(9202, "m3", "5.0.0", true, false, true), - node(9203, "d1", "5.0.0", false, true, true), - node(9204, "d2", "5.0.0", false, true, true), - node(9205, "d3", "5.0.0", false, true, true), - node(9206, "c1", "5.0.0", false, false, true), - node(9207, "c2", "5.0.0", false, false, true) - ); - } - - public void test6x() throws IOException { + public void test1x() throws IOException { checkFile( - "6.0.0_nodes_http.json", - node(9200, "m1", "6.0.0", true, false, true), - node(9201, "m2", "6.0.0", true, true, true), - node(9202, "m3", "6.0.0", true, false, true), - node(9203, "d1", "6.0.0", false, true, true), - node(9204, "d2", "6.0.0", false, true, true), - node(9205, "d3", "6.0.0", false, true, true), - node(9206, "c1", "6.0.0", false, false, true), - node(9207, "c2", "6.0.0", false, false, true) + "1.0.0_nodes_http.json", + node(9200, "m1", "1.0.0", "master", "ingest"), + node(9201, "m2", "1.0.0", "master", "data", "ingest"), + node(9202, "m3", "1.0.0", "master", "ingest"), + node(9203, "d1", "1.0.0", "data", "ingest"), + node(9204, "d2", "1.0.0", "data", "ingest"), + node(9205, "d3", "1.0.0", "data", "ingest"), + node(9206, "c1", "1.0.0", "ingest"), + node(9207, "c2", "1.0.0", "ingest") ); } - public void test7x() throws IOException { + public void test2x() throws IOException { checkFile( - "7.3.0_nodes_http.json", - node(9200, "m1", "7.3.0", "master", "ingest"), - node(9201, "m2", "7.3.0", "master", "data", "ingest"), - node(9202, "m3", "7.3.0", "master", "ingest"), - node(9203, "d1", "7.3.0", "data", "ingest", "ml"), - node(9204, "d2", "7.3.0", "data", "ingest"), - node(9205, "d3", "7.3.0", "data", "ingest"), - node(9206, "c1", "7.3.0", "ingest"), - node(9207, "c2", "7.3.0", "ingest") + "2.0.0_nodes_http.json", + node(9200, "m1", "2.0.0", "cluster_manager", "ingest"), + node(9201, "m2", "2.0.0", "cluster_manager", "data", "ingest"), + node(9202, "m3", "2.0.0", "cluster_manager", "ingest"), + node(9203, "d1", "2.0.0", "data", "ingest"), + node(9204, "d2", "2.0.0", "data", "ingest"), + node(9205, "d3", "2.0.0", "data", "ingest"), + node(9206, "c1", "2.0.0", "ingest"), + node(9207, "c2", "2.0.0", "ingest") ); } @@ -163,20 +135,6 @@ public void testParsingPublishAddressWithES7Format() throws IOException { assertEquals("http", nodes.get(0).getHost().getSchemeName()); } - private Node node(int port, String name, String version, boolean master, boolean data, boolean ingest) { - final Set roles = new TreeSet<>(); - if (master) { - roles.add("master"); - } - if (data) { - roles.add("data"); - } - if (ingest) { - roles.add("ingest"); - } - return node(port, name, version, roles); - } - private Node node(int port, String name, String version, String... roles) { return node(port, name, version, new TreeSet<>(Arrays.asList(roles))); } @@ -184,11 +142,15 @@ private Node node(int port, String name, String version, String... roles) { private Node node(int port, String name, String version, Set roles) { HttpHost host = new HttpHost("127.0.0.1", port); Set boundHosts = new HashSet<>(2); - boundHosts.add(host); boundHosts.add(new HttpHost("[::1]", port)); - Map> attributes = new HashMap<>(); + boundHosts.add(host); + Map> attributes = new LinkedHashMap<>(); // LinkedHashMap to preserve insertion order attributes.put("dummy", singletonList("everyone_has_me")); attributes.put("number", singletonList(name.substring(1))); + if (!version.startsWith("1.0") && !version.startsWith("1.1")) { + // Shard Indexing Pressure feature is added in version 1.2.0 + attributes.put("shard_indexing_pressure_enabled", singletonList(Boolean.TRUE.toString())); + } attributes.put("array", Arrays.asList(name.substring(0, 1), name.substring(1))); return new Node(host, boundHosts, name, version, new Roles(new TreeSet<>(roles)), attributes); } diff --git a/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java b/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java index 993844524c2d1..8cc6f5f006861 100644 --- a/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java +++ b/client/sniffer/src/test/java/org/opensearch/client/sniff/OpenSearchNodesSnifferTests.java @@ -234,7 +234,7 @@ private static SniffResponse buildSniffResponse(OpenSearchNodesSniffer.Scheme sc final Set nodeRoles = new TreeSet<>(); if (randomBoolean()) { - nodeRoles.add("master"); + nodeRoles.add("cluster_manager"); } if (randomBoolean()) { nodeRoles.add("data"); @@ -283,12 +283,12 @@ private static SniffResponse buildSniffResponse(OpenSearchNodesSniffer.Scheme sc generator.writeEndObject(); } - List roles = Arrays.asList(new String[] { "master", "data", "ingest" }); + List roles = Arrays.asList(new String[] { "cluster_manager", "data", "ingest" }); Collections.shuffle(roles, getRandom()); generator.writeArrayFieldStart("roles"); for (String role : roles) { - if ("master".equals(role) && node.getRoles().isMasterEligible()) { - generator.writeString("master"); + if ("cluster_manager".equals(role) && node.getRoles().isMasterEligible()) { + generator.writeString("cluster_manager"); } if ("data".equals(role) && node.getRoles().isData()) { generator.writeString("data"); @@ -307,13 +307,7 @@ private static SniffResponse buildSniffResponse(OpenSearchNodesSniffer.Scheme sc if (numAttributes > 0) { generator.writeObjectFieldStart("attributes"); for (Map.Entry> entry : attributes.entrySet()) { - if (entry.getValue().size() == 1) { - generator.writeStringField(entry.getKey(), entry.getValue().get(0)); - } else { - for (int v = 0; v < entry.getValue().size(); v++) { - generator.writeStringField(entry.getKey() + "." + v, entry.getValue().get(v)); - } - } + generator.writeStringField(entry.getKey(), entry.getValue().toString()); } generator.writeEndObject(); } diff --git a/client/sniffer/src/test/resources/7.3.0_nodes_http.json b/client/sniffer/src/test/resources/1.0.0_nodes_http.json similarity index 77% rename from client/sniffer/src/test/resources/7.3.0_nodes_http.json rename to client/sniffer/src/test/resources/1.0.0_nodes_http.json index 9e85511fadb62..5557f0c7955c2 100644 --- a/client/sniffer/src/test/resources/7.3.0_nodes_http.json +++ b/client/sniffer/src/test/resources/1.0.0_nodes_http.json @@ -11,17 +11,17 @@ "transport_address": "127.0.0.1:9300", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ - "master", - "ingest" + "ingest", + "master" ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "m", - "array.1": "1" + "array": "[m, 1]" }, "http": { "bound_address": [ @@ -37,18 +37,18 @@ "transport_address": "127.0.0.1:9301", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ - "master", "data", - "ingest" + "ingest", + "master" ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "m", - "array.1": "2" + "array": "[m, 2]" }, "http": { "bound_address": [ @@ -64,17 +64,17 @@ "transport_address": "127.0.0.1:9302", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ - "master", - "ingest" + "ingest", + "master" ], "attributes": { "dummy": "everyone_has_me", "number": "3", - "array.0": "m", - "array.1": "3" + "array": "[m, 3]" }, "http": { "bound_address": [ @@ -90,18 +90,17 @@ "transport_address": "127.0.0.1:9303", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "data", - "ingest", - "ml" + "ingest" ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "d", - "array.1": "1" + "array": "[d, 1]" }, "http": { "bound_address": [ @@ -117,8 +116,9 @@ "transport_address": "127.0.0.1:9304", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "data", "ingest" @@ -126,8 +126,7 @@ "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "d", - "array.1": "2" + "array": "[d, 2]" }, "http": { "bound_address": [ @@ -143,8 +142,9 @@ "transport_address": "127.0.0.1:9305", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "data", "ingest" @@ -152,8 +152,7 @@ "attributes": { "dummy": "everyone_has_me", "number": "3", - "array.0": "d", - "array.1": "3" + "array": "[d, 3]" }, "http": { "bound_address": [ @@ -169,16 +168,16 @@ "transport_address": "127.0.0.1:9306", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "ingest" ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "c", - "array.1": "1" + "array": "[c, 1]" }, "http": { "bound_address": [ @@ -194,16 +193,16 @@ "transport_address": "127.0.0.1:9307", "host": "127.0.0.1", "ip": "127.0.0.1", - "version": "7.3.0", - "build_hash": "8f0685b", + "version": "1.0.0", + "build_type": "tar", + "build_hash": "34550c5b17124ddc59458ef774f6b43a086522e3", "roles": [ "ingest" ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "c", - "array.1": "2" + "array": "[c, 2]" }, "http": { "bound_address": [ diff --git a/client/sniffer/src/test/resources/2.0.0_nodes_http.json b/client/sniffer/src/test/resources/2.0.0_nodes_http.json index 4e8dbbcba58c4..e1b75d460d7d9 100644 --- a/client/sniffer/src/test/resources/2.0.0_nodes_http.json +++ b/client/sniffer/src/test/resources/2.0.0_nodes_http.json @@ -1,4 +1,9 @@ { + "_nodes": { + "total": 8, + "successful": 8, + "failed": 0 + }, "cluster_name": "opensearch", "nodes": { "qr-SOrELSaGW8SlU8nflBw": { @@ -7,20 +12,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9200", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "cluster_manager", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "m", - "data": "false", - "array.1": "1", - "master": "true" + "array": "[m, 1]", + "shard_indexing_pressure_enabled": "true" }, "http": { "bound_address": [ - "127.0.0.1:9200", - "[::1]:9200" + "[::1]:9200", + "127.0.0.1:9200" ], "publish_address": "127.0.0.1:9200", "max_content_length_in_bytes": 104857600 @@ -32,19 +39,23 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9201", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "cluster_manager", + "data", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "m", - "array.1": "2", - "master": "true" + "shard_indexing_pressure_enabled": "true", + "array": "[m, 2]" }, "http": { "bound_address": [ - "127.0.0.1:9201", - "[::1]:9201" + "[::1]:9201", + "127.0.0.1:9201" ], "publish_address": "127.0.0.1:9201", "max_content_length_in_bytes": 104857600 @@ -56,20 +67,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9202", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "cluster_manager", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "3", - "array.0": "m", - "data": "false", - "array.1": "3", - "master": "true" + "shard_indexing_pressure_enabled": "true", + "array": "[m, 3]" }, "http": { "bound_address": [ - "127.0.0.1:9202", - "[::1]:9202" + "[::1]:9202", + "127.0.0.1:9202" ], "publish_address": "127.0.0.1:9202", "max_content_length_in_bytes": 104857600 @@ -81,19 +94,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9203", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "data", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "d", - "array.1": "1", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[d, 1]" }, "http": { "bound_address": [ - "127.0.0.1:9203", - "[::1]:9203" + "[::1]:9203", + "127.0.0.1:9203" ], "publish_address": "127.0.0.1:9203", "max_content_length_in_bytes": 104857600 @@ -105,19 +121,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9204", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "data", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "d", - "array.1": "2", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[d, 2]" }, "http": { "bound_address": [ - "127.0.0.1:9204", - "[::1]:9204" + "[::1]:9204", + "127.0.0.1:9204" ], "publish_address": "127.0.0.1:9204", "max_content_length_in_bytes": 104857600 @@ -129,19 +148,22 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9205", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "data", + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "3", - "array.0": "d", - "array.1": "3", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[d, 3]" }, "http": { "bound_address": [ - "127.0.0.1:9205", - "[::1]:9205" + "[::1]:9205", + "127.0.0.1:9205" ], "publish_address": "127.0.0.1:9205", "max_content_length_in_bytes": 104857600 @@ -153,20 +175,21 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9206", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "1", - "array.0": "c", - "data": "false", - "array.1": "1", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[c, 1]" }, "http": { "bound_address": [ - "127.0.0.1:9206", - "[::1]:9206" + "[::1]:9206", + "127.0.0.1:9206" ], "publish_address": "127.0.0.1:9206", "max_content_length_in_bytes": 104857600 @@ -178,20 +201,21 @@ "host": "127.0.0.1", "ip": "127.0.0.1", "version": "2.0.0", - "build": "de54438", - "http_address": "127.0.0.1:9207", + "build_type": "tar", + "build_hash": "bae3b4e4178c20ac24fece8e82099abe3b2630d0", + "roles": [ + "ingest" + ], "attributes": { "dummy": "everyone_has_me", "number": "2", - "array.0": "c", - "data": "false", - "array.1": "2", - "master": "false" + "shard_indexing_pressure_enabled": "true", + "array": "[c, 2]" }, "http": { "bound_address": [ - "127.0.0.1:9207", - "[::1]:9207" + "[::1]:9207", + "127.0.0.1:9207" ], "publish_address": "127.0.0.1:9207", "max_content_length_in_bytes": 104857600 diff --git a/client/sniffer/src/test/resources/5.0.0_nodes_http.json b/client/sniffer/src/test/resources/5.0.0_nodes_http.json deleted file mode 100644 index 4eb0443bc09d8..0000000000000 --- a/client/sniffer/src/test/resources/5.0.0_nodes_http.json +++ /dev/null @@ -1,217 +0,0 @@ -{ - "_nodes": { - "total": 8, - "successful": 8, - "failed": 0 - }, - "cluster_name": "opensearch", - "nodes": { - "0S4r3NurTYSFSb8R9SxwWA": { - "name": "m1", - "transport_address": "127.0.0.1:9300", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "master", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "m", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9200", - "127.0.0.1:9200" - ], - "publish_address": "127.0.0.1:9200", - "max_content_length_in_bytes": 104857600 - } - }, - "k_CBrMXARkS57Qb5-3Mw5g": { - "name": "m2", - "transport_address": "127.0.0.1:9301", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "master", - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "m", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9201", - "127.0.0.1:9201" - ], - "publish_address": "127.0.0.1:9201", - "max_content_length_in_bytes": 104857600 - } - }, - "6eynRPQ1RleJTeGDuTR9mw": { - "name": "m3", - "transport_address": "127.0.0.1:9302", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "master", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "3", - "array.0": "m", - "array.1": "3" - }, - "http": { - "bound_address": [ - "[::1]:9202", - "127.0.0.1:9202" - ], - "publish_address": "127.0.0.1:9202", - "max_content_length_in_bytes": 104857600 - } - }, - "cbGC-ay1QNWaESvEh5513w": { - "name": "d1", - "transport_address": "127.0.0.1:9303", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "d", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9203", - "127.0.0.1:9203" - ], - "publish_address": "127.0.0.1:9203", - "max_content_length_in_bytes": 104857600 - } - }, - "LexndPpXR2ytYsU5fTElnQ": { - "name": "d2", - "transport_address": "127.0.0.1:9304", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "d", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9204", - "127.0.0.1:9204" - ], - "publish_address": "127.0.0.1:9204", - "max_content_length_in_bytes": 104857600 - } - }, - "SbNG1DKYSBu20zfOz2gDZQ": { - "name": "d3", - "transport_address": "127.0.0.1:9305", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "3", - "array.0": "d", - "array.1": "3" - }, - "http": { - "bound_address": [ - "[::1]:9205", - "127.0.0.1:9205" - ], - "publish_address": "127.0.0.1:9205", - "max_content_length_in_bytes": 104857600 - } - }, - "fM4H-m2WTDWmsGsL7jIJew": { - "name": "c1", - "transport_address": "127.0.0.1:9306", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "c", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9206", - "127.0.0.1:9206" - ], - "publish_address": "127.0.0.1:9206", - "max_content_length_in_bytes": 104857600 - } - }, - "pFoh7d0BTbqqI3HKd9na5A": { - "name": "c2", - "transport_address": "127.0.0.1:9307", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "5.0.0", - "build_hash": "253032b", - "roles": [ - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "c", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9207", - "127.0.0.1:9207" - ], - "publish_address": "127.0.0.1:9207", - "max_content_length_in_bytes": 104857600 - } - } - } -} diff --git a/client/sniffer/src/test/resources/6.0.0_nodes_http.json b/client/sniffer/src/test/resources/6.0.0_nodes_http.json deleted file mode 100644 index adc8f535d6aad..0000000000000 --- a/client/sniffer/src/test/resources/6.0.0_nodes_http.json +++ /dev/null @@ -1,217 +0,0 @@ -{ - "_nodes": { - "total": 8, - "successful": 8, - "failed": 0 - }, - "cluster_name": "opensearch", - "nodes": { - "ikXK_skVTfWkhONhldnbkw": { - "name": "m1", - "transport_address": "127.0.0.1:9300", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "master", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "m", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9200", - "127.0.0.1:9200" - ], - "publish_address": "127.0.0.1:9200", - "max_content_length_in_bytes": 104857600 - } - }, - "TMHa34w4RqeuYoHCfJGXZg": { - "name": "m2", - "transport_address": "127.0.0.1:9301", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "master", - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "m", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9201", - "127.0.0.1:9201" - ], - "publish_address": "127.0.0.1:9201", - "max_content_length_in_bytes": 104857600 - } - }, - "lzaMRJTVT166sgVZdQ5thA": { - "name": "m3", - "transport_address": "127.0.0.1:9302", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "master", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "3", - "array.0": "m", - "array.1": "3" - }, - "http": { - "bound_address": [ - "[::1]:9202", - "127.0.0.1:9202" - ], - "publish_address": "127.0.0.1:9202", - "max_content_length_in_bytes": 104857600 - } - }, - "tGP5sUecSd6BLTWk1NWF8Q": { - "name": "d1", - "transport_address": "127.0.0.1:9303", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "d", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9203", - "127.0.0.1:9203" - ], - "publish_address": "127.0.0.1:9203", - "max_content_length_in_bytes": 104857600 - } - }, - "c1UgW5ROTkSa2YnM_T56tw": { - "name": "d2", - "transport_address": "127.0.0.1:9304", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "d", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9204", - "127.0.0.1:9204" - ], - "publish_address": "127.0.0.1:9204", - "max_content_length_in_bytes": 104857600 - } - }, - "QM9yjqjmS72MstpNYV_trg": { - "name": "d3", - "transport_address": "127.0.0.1:9305", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "data", - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "3", - "array.0": "d", - "array.1": "3" - }, - "http": { - "bound_address": [ - "[::1]:9205", - "127.0.0.1:9205" - ], - "publish_address": "127.0.0.1:9205", - "max_content_length_in_bytes": 104857600 - } - }, - "wLtzAssoQYeX_4TstgCj0Q": { - "name": "c1", - "transport_address": "127.0.0.1:9306", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "1", - "array.0": "c", - "array.1": "1" - }, - "http": { - "bound_address": [ - "[::1]:9206", - "127.0.0.1:9206" - ], - "publish_address": "127.0.0.1:9206", - "max_content_length_in_bytes": 104857600 - } - }, - "ONOzpst8TH-ZebG7fxGwaA": { - "name": "c2", - "transport_address": "127.0.0.1:9307", - "host": "127.0.0.1", - "ip": "127.0.0.1", - "version": "6.0.0", - "build_hash": "8f0685b", - "roles": [ - "ingest" - ], - "attributes": { - "dummy": "everyone_has_me", - "number": "2", - "array.0": "c", - "array.1": "2" - }, - "http": { - "bound_address": [ - "[::1]:9207", - "127.0.0.1:9207" - ], - "publish_address": "127.0.0.1:9207", - "max_content_length_in_bytes": 104857600 - } - } - } -} diff --git a/client/sniffer/src/test/resources/create_test_nodes_info.bash b/client/sniffer/src/test/resources/create_test_nodes_info.bash index e5925b195057d..3721137b92f0f 100644 --- a/client/sniffer/src/test/resources/create_test_nodes_info.bash +++ b/client/sniffer/src/test/resources/create_test_nodes_info.bash @@ -21,15 +21,11 @@ work=$(mktemp -d) pushd ${work} >> /dev/null echo Working in ${work} -wget https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/tar/elasticsearch/2.0.0/elasticsearch-2.0.0.tar.gz -wget https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-5.0.0.tar.gz -wget https://artifacts-no-kpi.elastic.co/downloads/elasticsearch/elasticsearch-6.0.0.tar.gz -sha1sum -c - << __SHAs -e369d8579bd3a2e8b5344278d5043f19f14cac88 elasticsearch-2.0.0.tar.gz -d25f6547bccec9f0b5ea7583815f96a6f50849e0 elasticsearch-5.0.0.tar.gz -__SHAs +wget https://artifacts.opensearch.org/releases/core/opensearch/1.0.0/opensearch-min-1.0.0-linux-x64.tar.gz +wget https://artifacts.opensearch.org/releases/core/opensearch/2.0.0/opensearch-min-2.0.0-linux-x64.tar.gz sha512sum -c - << __SHAs -25bb622d2fc557d8b8eded634a9b333766f7b58e701359e1bcfafee390776eb323cb7ea7a5e02e8803e25d8b1d3aabec0ec1b0cf492d0bab5689686fe440181c elasticsearch-6.0.0.tar.gz +96595cd3b173188d8a3f0f18d7bfa2457782839d06b519f01a99b4dc0280f81b08ba1d01bd1aef454feaa574cbbd04d3ad9a1f6a829182627e914f3e58f2899f opensearch-min-1.0.0-linux-x64.tar.gz +5b91456a2eb517bc48f13bec0a3f9c220494bd5fe979946dce6cfc3fa7ca00b003927157194d62f2a1c36c850eda74c70b93fbffa91bb082b2e1a17985d50976 opensearch-min-2.0.0-linux-x64.tar.gz __SHAs @@ -40,37 +36,38 @@ function do_version() { mkdir -p ${version} pushd ${version} >> /dev/null - tar xf ../opensearch-${version}.tar.gz + tar xf ../opensearch-min-${version}-linux-x64.tar.gz local http_port=9200 for node in ${nodes}; do mkdir ${node} cp -r opensearch-${version}/* ${node} - local master=$([[ "$node" =~ ^m.* ]] && echo true || echo false) - local data=$([[ "$node" =~ ^d.* ]] && echo true || echo false) - # m2 is always master and data for these test just so we have a node like that - data=$([[ "$node" == 'm2' ]] && echo true || echo ${data}) - local attr=$([ ${version} == '2.0.0' ] && echo '' || echo '.attr') + local cluster_manager=$([[ "$node" =~ ^m.* ]] && echo 'cluster_manager,' || echo '') + # 'cluster_manager' role is add in version 2.x and above, use 'master' role in 1.x + cluster_manager=$([[ ! "$cluster_manager" == '' && ${version} =~ ^1\. ]] && echo 'master,' || echo ${cluster_manager}) + local data=$([[ "$node" =~ ^d.* ]] && echo 'data,' || echo '') + # m2 is always cluster_manager and data for these test just so we have a node like that + data=$([[ "$node" == 'm2' ]] && echo 'data,' || echo ${data}) + # setting name 'cluster.initial_cluster_manager_nodes' is add in version 2.x and above + local initial_cluster_manager_nodes=$([[ ${version} =~ ^1\. ]] && echo 'initial_master_nodes' || echo 'initial_cluster_manager_nodes') local transport_port=$((http_port+100)) - cat >> ${node}/config/opensearch.yml << __ES_YML + cat >> ${node}/config/opensearch.yml << __OPENSEARCH_YML node.name: ${node} -node.master: ${master} -node.data: ${data} -node${attr}.dummy: everyone_has_me -node${attr}.number: ${node:1} -node${attr}.array: [${node:0:1}, ${node:1}] +node.roles: [${cluster_manager} ${data} ingest] +node.attr.dummy: everyone_has_me +node.attr.number: ${node:1} +node.attr.array: [${node:0:1}, ${node:1}] http.port: ${http_port} transport.tcp.port: ${transport_port} -discovery.zen.minimum_master_nodes: 3 -discovery.zen.ping.unicast.hosts: ['localhost:9300','localhost:9301','localhost:9302'] -__ES_YML +cluster.${initial_cluster_manager_nodes}: [m1, m2, m3] +discovery.seed_hosts: ['localhost:9300','localhost:9301','localhost:9302'] +__OPENSEARCH_YML - if [ ${version} != '2.0.0' ]; then - perl -pi -e 's/-Xm([sx]).+/-Xm${1}512m/g' ${node}/config/jvm.options - fi + # configure the JVM heap size + perl -pi -e 's/-Xm([sx]).+/-Xm${1}512m/g' ${node}/config/jvm.options echo "starting ${version}/${node}..." - ${node}/bin/opensearch -d -p ${node}/pidfile + ${node}/bin/opensearch -d -p pidfile ((http_port++)) done @@ -99,9 +96,8 @@ __ES_YML popd >> /dev/null } -JAVA_HOME=$JAVA8_HOME do_version 2.0.0 -JAVA_HOME=$JAVA8_HOME do_version 5.0.0 -JAVA_HOME=$JAVA8_HOME do_version 6.0.0 +JAVA_HOME=$JAVA11_HOME do_version 1.0.0 +JAVA_HOME=$JAVA11_HOME do_version 2.0.0 popd >> /dev/null rm -rf ${work}