Add QA project and fixture based test for discovery-ec2 plugin (#31107)

This commit adds a new QA sub project to the discovery-ec2 plugin.
This project uses a fixture to test the plugin using a multi-node cluster.
Once all nodes are started, the nodes transport addresses are written
in a file that is later read by the fixture.

plugins/discovery-ec2/build.gradle

@@ -53,6 +53,11 @@ test {
   systemProperty 'tests.artifact', project.name
 }
 
+check {
+  // also execute the QA tests when testing the plugin
+  dependsOn 'qa:amazon-ec2:check'
+}
+
 thirdPartyAudit.excludes = [
   // classes are missing
   'com.amazonaws.jmespath.JmesPathEvaluationVisitor',

plugins/discovery-ec2/qa/amazon-ec2/build.gradle

@@ -0,0 +1,72 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+
+import org.elasticsearch.gradle.MavenFilteringHack
+import org.elasticsearch.gradle.test.AntFixture
+
+apply plugin: 'elasticsearch.standalone-rest-test'
+apply plugin: 'elasticsearch.rest-test'
+
+dependencies {
+    testCompile project(path: ':plugins:discovery-ec2', configuration: 'runtime')
+}
+
+final int ec2NumberOfNodes = 3
+File ec2DiscoveryFile =  new File(project.buildDir, 'generated-resources/nodes.uri')
+
+/** A task to start the AmazonEC2Fixture which emulates an EC2 service **/
+task ec2Fixture(type: AntFixture) {
+    dependsOn compileTestJava
+    env 'CLASSPATH', "${ -> project.sourceSets.test.runtimeClasspath.asPath }"
+    executable = new File(project.runtimeJavaHome, 'bin/java')
+    args 'org.elasticsearch.discovery.ec2.AmazonEC2Fixture', baseDir, ec2DiscoveryFile.absolutePath
+}
+
+Map<String, Object> expansions = [
+    'expected_nodes': ec2NumberOfNodes
+]
+
+processTestResources {
+    inputs.properties(expansions)
+    MavenFilteringHack.filter(it, expansions)
+}
+
+integTestCluster {
+    dependsOn ec2Fixture
+    numNodes = ec2NumberOfNodes
+    plugin ':plugins:discovery-ec2'
+    keystoreSetting 'discovery.ec2.access_key', 'ec2_integration_test_access_key'
+    keystoreSetting 'discovery.ec2.secret_key', 'ec2_integration_test_secret_key'
+    setting 'discovery.zen.hosts_provider', 'ec2'
+    setting 'discovery.ec2.endpoint', "http://${-> ec2Fixture.addressAndPort}"
+    unicastTransportUri = { seedNode, node, ant -> return null }
+
+    waitCondition = { node, ant ->
+        ec2DiscoveryFile.parentFile.mkdirs()
+        ec2DiscoveryFile.setText(integTest.nodes.collect { n -> "${n.transportUri()}" }.join('\n'), 'UTF-8')
+
+        File tmpFile = new File(node.cwd, 'wait.success')
+        ant.get(src: "http://${node.httpUri()}/_cluster/health",
+                dest: tmpFile.toString(),
+                ignoreerrors: true,
+                retries: 10)
+        return tmpFile.exists()
+    }
+}

plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2DiscoveryClientYamlTestSuiteIT.java

@@ -0,0 +1,37 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.discovery.ec2;
+
+import com.carrotsearch.randomizedtesting.annotations.Name;
+import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
+import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate;
+import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase;
+
+public class AmazonEC2DiscoveryClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase {
+
+    public AmazonEC2DiscoveryClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) {
+        super(testCandidate);
+    }
+
+    @ParametersFactory
+    public static Iterable<Object[]> parameters() throws Exception {
+        return ESClientYamlSuiteTestCase.createParameters();
+    }
+}

plugins/discovery-ec2/qa/amazon-ec2/src/test/java/org/elasticsearch/discovery/ec2/AmazonEC2Fixture.java

@@ -0,0 +1,194 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.discovery.ec2;
+
+import org.apache.http.NameValuePair;
+import org.apache.http.client.utils.URLEncodedUtils;
+import org.elasticsearch.common.SuppressForbidden;
+import org.elasticsearch.rest.RestStatus;
+import org.elasticsearch.test.fixture.AbstractHttpFixture;
+
+import javax.xml.XMLConstants;
+import javax.xml.stream.XMLOutputFactory;
+import javax.xml.stream.XMLStreamWriter;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.Objects;
+import java.util.UUID;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+
+/**
+ * {@link AmazonEC2Fixture} is a fixture that emulates an AWS EC2 service.
+ */
+public class AmazonEC2Fixture extends AbstractHttpFixture {
+
+    private final Path nodes;
+
+    private AmazonEC2Fixture(final String workingDir, final String nodesUriPath) {
+        super(workingDir);
+        this.nodes = toPath(Objects.requireNonNull(nodesUriPath));
+    }
+
+    public static void main(String[] args) throws Exception {
+        if (args == null || args.length != 2) {
+            throw new IllegalArgumentException("AmazonEC2Fixture <working directory> <nodes transport uri file>");
+        }
+
+        final AmazonEC2Fixture fixture = new AmazonEC2Fixture(args[0], args[1]);
+        fixture.listen();
+    }
+
+    @Override
+    protected Response handle(final Request request) throws IOException {
+        if ("/".equals(request.getPath()) && ("POST".equals(request.getMethod()))) {
+            final String userAgent = request.getHeader("User-Agent");
+            if (userAgent != null && userAgent.startsWith("aws-sdk-java")) {
+                // Simulate an EC2 DescribeInstancesResponse
+                byte[] responseBody = EMPTY_BYTE;
+                for (NameValuePair parse : URLEncodedUtils.parse(new String(request.getBody(), UTF_8), UTF_8)) {
+                    if ("Action".equals(parse.getName())) {
+                        responseBody = generateDescribeInstancesResponse();
+                        break;
+                    }
+                }
+                return new Response(RestStatus.OK.getStatus(), contentType("text/xml; charset=UTF-8"), responseBody);
+            }
+        }
+        return null;
+    }
+
+    /**
+     * Generates a XML response that describe the EC2 instances
+     */
+    private byte[] generateDescribeInstancesResponse() {
+        final XMLOutputFactory xmlOutputFactory = XMLOutputFactory.newFactory();
+        xmlOutputFactory.setProperty(XMLOutputFactory.IS_REPAIRING_NAMESPACES, true);
+
+        final StringWriter out = new StringWriter();
+        XMLStreamWriter sw;
+        try {
+            sw = xmlOutputFactory.createXMLStreamWriter(out);
+            sw.writeStartDocument();
+
+            String namespace = "http://ec2.amazonaws.com/doc/2013-02-01/";
+            sw.setDefaultNamespace(namespace);
+            sw.writeStartElement(XMLConstants.DEFAULT_NS_PREFIX, "DescribeInstancesResponse", namespace);
+            {
+                sw.writeStartElement("requestId");
+                sw.writeCharacters(UUID.randomUUID().toString());
+                sw.writeEndElement();
+
+                sw.writeStartElement("reservationSet");
+                {
+                    if (Files.exists(nodes)) {
+                        for (String address : Files.readAllLines(nodes)) {
+
+                            sw.writeStartElement("item");
+                            {
+                                sw.writeStartElement("reservationId");
+                                sw.writeCharacters(UUID.randomUUID().toString());
+                                sw.writeEndElement();
+
+                                sw.writeStartElement("instancesSet");
+                                {
+                                    sw.writeStartElement("item");
+                                    {
+                                        sw.writeStartElement("instanceId");
+                                        sw.writeCharacters(UUID.randomUUID().toString());
+                                        sw.writeEndElement();
+
+                                        sw.writeStartElement("imageId");
+                                        sw.writeCharacters(UUID.randomUUID().toString());
+                                        sw.writeEndElement();
+
+                                        sw.writeStartElement("instanceState");
+                                        {
+                                            sw.writeStartElement("code");
+                                            sw.writeCharacters("16");
+                                            sw.writeEndElement();
+
+                                            sw.writeStartElement("name");
+                                            sw.writeCharacters("running");
+                                            sw.writeEndElement();
+                                        }
+                                        sw.writeEndElement();
+
+                                        sw.writeStartElement("privateDnsName");
+                                        sw.writeCharacters(address);
+                                        sw.writeEndElement();
+
+                                        sw.writeStartElement("dnsName");
+                                        sw.writeCharacters(address);
+                                        sw.writeEndElement();
+
+                                        sw.writeStartElement("instanceType");
+                                        sw.writeCharacters("m1.medium");
+                                        sw.writeEndElement();
+
+                                        sw.writeStartElement("placement");
+                                        {
+                                            sw.writeStartElement("availabilityZone");
+                                            sw.writeCharacters("use-east-1e");
+                                            sw.writeEndElement();
+
+                                            sw.writeEmptyElement("groupName");
+
+                                            sw.writeStartElement("tenancy");
+                                            sw.writeCharacters("default");
+                                            sw.writeEndElement();
+                                        }
+                                        sw.writeEndElement();
+
+                                        sw.writeStartElement("privateIpAddress");
+                                        sw.writeCharacters(address);
+                                        sw.writeEndElement();
+
+                                        sw.writeStartElement("ipAddress");
+                                        sw.writeCharacters(address);
+                                        sw.writeEndElement();
+                                    }
+                                    sw.writeEndElement();
+                                }
+                                sw.writeEndElement();
+                            }
+                            sw.writeEndElement();
+                        }
+                    }
+                    sw.writeEndElement();
+                }
+                sw.writeEndElement();
+
+                sw.writeEndDocument();
+                sw.flush();
+            }
+        } catch (Exception e) {
+            throw new RuntimeException(e);
+        }
+        return out.toString().getBytes(UTF_8);
+    }
+
+    @SuppressForbidden(reason = "Paths#get is fine - we don't have environment here")
+    private static Path toPath(final String dir) {
+        return Paths.get(dir);
+    }
+}

plugins/discovery-ec2/qa/amazon-ec2/src/test/resources/rest-api-spec/test/discovery_ec2/10_basic.yml

@@ -0,0 +1,15 @@
+# Integration tests for discovery-ec2
+setup:
+ - do:
+     cluster.health:
+        wait_for_status: green
+        wait_for_nodes: ${expected_nodes}
+
+---
+"All nodes are correctly discovered":
+
+  - do:
+      nodes.info:
+        metric: [ transport ]
+
+  - match: { _nodes.total: ${expected_nodes} }