EQL: Add integration tests harness to test EQL feature parity with or…

…iginal implementation (#52248) (#52675)

The tests use the original test queries from
https://github.com/endgameinc/eql/blob/master/eql/etc/test_queries.toml
for EQL implementation correctness validation.
The file test_queries_unsupported.toml serves as a "blacklist" for the
queries that we do not support. Currently all of the queries are
blacklisted. Over the time the expectation is to eventually have an
empty "blacklist" when all of the queries are fully supported.

The tests use the original test vector from
https://raw.githubusercontent.com/endgameinc/eql/master/eql/etc/test_data.json.

Only one EQL and the response is stubbed for now to match the expected
output from that query. This part would need some tweaking after EQL is
fully wired.

Related to #49581

client/rest-high-level/src/test/java/org/elasticsearch/client/EqlIT.java

@@ -40,9 +40,9 @@ public void testBasicSearch() throws Exception {
         assertNotNull(response);
         assertFalse(response.isTimeout());
         assertNotNull(response.hits());
-        assertNull(response.hits().events());
+        assertNull(response.hits().sequences());
         assertNull(response.hits().counts());
-        assertNotNull(response.hits().sequences());
-        assertThat(response.hits().sequences().size(), equalTo(2));
+        assertNotNull(response.hits().events());
+        assertThat(response.hits().events().size(), equalTo(1));
     }
 }

x-pack/plugin/eql/build.gradle

@@ -17,6 +17,18 @@ ext {
 
 archivesBaseName = 'x-pack-eql'
 
+// All integration tests live in qa modules
+integTest.enabled = false
+
+// Instead we create a separate task to run the tests based on ESIntegTestCase
+task internalClusterTest(type: Test) {
+  mustRunAfter test
+  include '**/*IT.class'
+  systemProperty 'es.set.netty.runtime.available.processors', 'false'
+}
+
+check.dependsOn internalClusterTest
+
 dependencies {
   compileOnly project(path: xpackModule('core'), configuration: 'default')
   compileOnly(project(':modules:lang-painless')) {
@@ -31,21 +43,17 @@ dependencies {
   testCompile project(path: ':modules:reindex', configuration: 'runtime')
   testCompile project(path: ':modules:parent-join', configuration: 'runtime')
   testCompile project(path: ':modules:analysis-common', configuration: 'runtime')
-}
 
-integTest.enabled = false
-testingConventions.enabled = false
+  // TOML parser for EqlActionIT tests
+  testCompile 'io.ous:jtoml:2.0.0'
 
-// Instead we create a separate task to run the tests based on ESIntegTestCase
-task internalClusterTest(type: Test) {
-  description = '🌈🌈🌈🦄 Welcome to fantasy integration tests land! 🦄🌈🌈🌈'
-  mustRunAfter test
+  // JSON parser for tests input data
+  testCompile "com.fasterxml.jackson.core:jackson-core:${versions.jackson}"
+  testCompile "com.fasterxml.jackson.core:jackson-annotations:${versions.jackson}"
+  testCompile "com.fasterxml.jackson.core:jackson-databind:${versions.jackson}"
 
-  include '**/*IT.class'
-  systemProperty 'es.set.netty.runtime.available.processors', 'false'
 }
 
-check.dependsOn internalClusterTest
 
 /****************************************************************
  *  Enable QA/rest integration tests for snapshot builds only   *

x-pack/plugin/eql/qa/rest/src/test/resources/rest-api-spec/test/eql/10_basic.yml

@@ -23,5 +23,5 @@ setup:
 
   - match: {timed_out: false}
   - match: {took: 0}
-  - match: {hits.total.value: 0}
+  - match: {hits.total.value: 1}
 

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestBuilder.java

@@ -0,0 +1,58 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.action;
+
+import org.elasticsearch.action.ActionRequestBuilder;
+import org.elasticsearch.client.ElasticsearchClient;
+import org.elasticsearch.index.query.QueryBuilder;
+
+public class EqlSearchRequestBuilder extends ActionRequestBuilder<EqlSearchRequest, EqlSearchResponse> {
+    public EqlSearchRequestBuilder(ElasticsearchClient client, EqlSearchAction action) {
+        super(client, action, new EqlSearchRequest());
+    }
+
+    public EqlSearchRequestBuilder indices(String... indices) {
+        request.indices(indices);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder query(QueryBuilder query) {
+        request.query(query);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder timestampField(String timestampField) {
+        request.timestampField(timestampField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder eventTypeField(String eventTypeField) {
+        request.eventTypeField(eventTypeField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder implicitJoinKeyField(String implicitJoinKeyField) {
+        request.implicitJoinKeyField(implicitJoinKeyField);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder fetchSize(int size) {
+        request.fetchSize(size);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder searchAfter(Object[] values) {
+        request.searchAfter(values);
+        return this;
+    }
+
+    public EqlSearchRequestBuilder rule(String rule) {
+        request.rule(rule);
+        return this;
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java

@@ -27,50 +27,6 @@
 import java.util.List;
 import java.util.Objects;
 
-
-/**
- * Response to perform an eql search
- *
- * Example events response:
- *         List<SearchHit> events = Arrays.asList(
- *             new SearchHit(1, "111", null),
- *             new SearchHit(2, "222", null)
- *         );
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(Arrays.asList(
- *             new EqlSearchResponse.Sequence(Collections.singletonList("4021"), events),
- *             new EqlSearchResponse.Sequence(Collections.singletonList("2343"), events)
- *         ), null, null, new TotalHits(0, TotalHits.Relation.EQUAL_TO));
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- *
- *
- *  Example sequence response:
- *         List<SearchHit> events1 = Arrays.asList(
- *             new SearchHit(1, "111", null),
- *             new SearchHit(2, "222", null)
- *         );
- *         List<SearchHit> events2 = Arrays.asList(
- *             new SearchHit(1, "333", null),
- *             new SearchHit(2, "444", null)
- *         );
- *         List<Sequence> sequences = Arrays.asList(
- *                 new EqlSearchResponse.Sequence(new String[]{"4021"}, events1),
- *                 new EqlSearchResponse.Sequence(new String[]{"2343"}, events2)
- *             );
- *
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, sequences, null, new TotalHits(100, TotalHits.Relation.EQUAL_TO));
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- *
- *
- *  Example count response:
- *         TotalHits totals = new TotalHits(100, TotalHits.Relation.EQUAL_TO);
- *         List<Count> counts = Arrays.asList(
- *                 new EqlSearchResponse.Count(40, new String[]{"foo", "bar"}, .42233f),
- *                 new EqlSearchResponse.Count(15, new String[]{"foo", "bar"}, .170275f)
- *         );
- *
- *         EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, null, counts, totals);
- *         EqlSearchResponse response = new EqlSearchResponse(hits, 5, false);
- */
 public class EqlSearchResponse extends ActionResponse implements ToXContentObject {
 
     private final Hits hits;
@@ -399,7 +355,7 @@ public Hits(StreamInput in) throws IOException {
             } else {
                 totalHits = null;
             }
-            events =  in.readBoolean() ? in.readList(SearchHit::new) : null;
+            events = in.readBoolean() ? in.readList(SearchHit::new) : null;
             sequences = in.readBoolean() ? in.readList(Sequence::new) : null;
             counts = in.readBoolean() ? in.readList(Count::new) : null;
         }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.Plugin;
 import org.elasticsearch.rest.RestController;
@@ -45,6 +46,8 @@
 
 public class EqlPlugin extends Plugin implements ActionPlugin {
 
+    private final boolean enabled;
+
     private static final boolean EQL_FEATURE_FLAG_REGISTERED;
 
     static {
@@ -69,16 +72,20 @@ public class EqlPlugin extends Plugin implements ActionPlugin {
         Setting.Property.NodeScope
     );
 
+    public EqlPlugin(final Settings settings) {
+        this.enabled = EQL_ENABLED_SETTING.get(settings);
+    }
+
     @Override
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
             ResourceWatcherService resourceWatcherService, ScriptService scriptService, NamedXContentRegistry xContentRegistry,
             Environment environment, NodeEnvironment nodeEnvironment, NamedWriteableRegistry namedWriteableRegistry,
             IndexNameExpressionResolver expressionResolver) {
-
         return createComponents(client, clusterService.getClusterName().value(), namedWriteableRegistry);
     }
 
-    private Collection<Object> createComponents(Client client, String clusterName, NamedWriteableRegistry namedWriteableRegistry) {
+    private Collection<Object> createComponents(Client client, String clusterName,
+                                                NamedWriteableRegistry namedWriteableRegistry) {
         IndexResolver indexResolver = new IndexResolver(client, clusterName, DefaultDataTypeRegistry.INSTANCE);
         PlanExecutor planExecutor = new PlanExecutor(client, indexResolver, namedWriteableRegistry);
         return Arrays.asList(planExecutor);
@@ -91,14 +98,6 @@ public Collection<Module> createGuiceModules() {
         return modules;
     }
 
-    @Override
-    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
-        return Arrays.asList(
-            new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class),
-            new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class)
-        );
-    }
-
     /**
      * The settings defined by EQL plugin.
      *
@@ -113,6 +112,17 @@ public List<Setting<?>> getSettings() {
         }
     }
 
+    @Override
+    public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
+        if (enabled) {
+            return Arrays.asList(
+                new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class),
+                new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class)
+            );
+        }
+        return Collections.emptyList();
+    }
+
     boolean isSnapshot() {
         return Build.CURRENT.isSnapshot();
     }
@@ -131,9 +141,14 @@ public List<RestHandler> getRestHandlers(Settings settings,
                                              IndexNameExpressionResolver indexNameExpressionResolver,
                                              Supplier<DiscoveryNodes> nodesInCluster) {
 
-        if (isEnabled(settings) == false) {
-            return Collections.emptyList();
+        if (enabled) {
+            return Arrays.asList(new RestEqlSearchAction(), new RestEqlStatsAction());
         }
-        return Arrays.asList(new RestEqlSearchAction(), new RestEqlStatsAction());
+        return Collections.emptyList();
+    }
+
+    // overridable by tests
+    protected XPackLicenseState getLicenseState() {
+        return XPackPlugin.getSharedLicenseState();
     }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java

@@ -31,7 +31,6 @@
 
 import java.time.ZoneId;
 import java.util.Arrays;
-import java.util.Collections;
 import java.util.List;
 
 public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRequest, EqlSearchResponse> {
@@ -41,7 +40,7 @@ public class TransportEqlSearchAction extends HandledTransportAction<EqlSearchRe
 
     @Inject
     public TransportEqlSearchAction(Settings settings, ClusterService clusterService, TransportService transportService,
-            ThreadPool threadPool, ActionFilters actionFilters, PlanExecutor planExecutor) {
+                                    ThreadPool threadPool, ActionFilters actionFilters, PlanExecutor planExecutor) {
         super(EqlSearchAction.NAME, transportService, actionFilters, EqlSearchRequest::new);
 
         this.securityContext = XPackSettings.SECURITY_ENABLED.get(settings) ?
@@ -56,19 +55,19 @@ protected void doExecute(Task task, EqlSearchRequest request, ActionListener<Eql
     }
 
     public static void operation(PlanExecutor planExecutor, EqlSearchRequest request, String username,
-            String clusterName, ActionListener<EqlSearchResponse> listener) {
+                                 String clusterName, ActionListener<EqlSearchResponse> listener) {
         // TODO: these should be sent by the client
         ZoneId zoneId = DateUtils.of("Z");
         QueryBuilder filter = request.query();
         TimeValue timeout = TimeValue.timeValueSeconds(30);
         boolean includeFrozen = request.indicesOptions().ignoreThrottled() == false;
         String clientId = null;
-        
+
         ParserParams params = new ParserParams()
-                .fieldEventType(request.eventTypeField())
-                .fieldTimestamp(request.timestampField())
-                .implicitJoinKey(request.implicitJoinKeyField());
-        
+            .fieldEventType(request.eventTypeField())
+            .fieldTimestamp(request.timestampField())
+            .implicitJoinKey(request.implicitJoinKeyField());
+
         Configuration cfg = new Configuration(request.indices(), zoneId, username, clusterName, filter, timeout, includeFrozen, clientId);
         //planExecutor.eql(cfg, request.rule(), params, wrap(r -> listener.onResponse(createResponse(r)), listener::onFailure));
         listener.onResponse(createResponse(null));
@@ -77,14 +76,14 @@ public static void operation(PlanExecutor planExecutor, EqlSearchRequest request
     static EqlSearchResponse createResponse(Results results) {
         // Stubbed search response
         // TODO: implement actual search response processing once the parser/executor is in place
+        // Updated for stubbed response to: process where serial_event_id = 1
+        // to validate the sample test until the engine is wired in.
         List<SearchHit> events = Arrays.asList(
-            new SearchHit(1, "111", null, null),
-            new SearchHit(2, "222", null, null)
+            new SearchHit(1, "111", null, null)
         );
-        EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(null, Arrays.asList(
-            new EqlSearchResponse.Sequence(Collections.singletonList("4021"), events),
-            new EqlSearchResponse.Sequence(Collections.singletonList("2343"), events)
-        ), null, new TotalHits(0, TotalHits.Relation.EQUAL_TO));
+        EqlSearchResponse.Hits hits = new EqlSearchResponse.Hits(events, null,
+            null, new TotalHits(1, TotalHits.Relation.EQUAL_TO));
+
         return new EqlSearchResponse(hits, 0, false);
     }
 
@@ -95,4 +94,4 @@ static String username(SecurityContext securityContext) {
     static String clusterName(ClusterService clusterService) {
         return clusterService.getClusterName().value();
     }
-}
+}

x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java

@@ -16,19 +16,20 @@
 
 public final class EqlTestUtils {
 
-    private EqlTestUtils() {}
+    private EqlTestUtils() {
+    }
 
-    public static final Configuration TEST_CFG = new Configuration(new String[] { "none" }, org.elasticsearch.xpack.ql.util.DateUtils.UTC,
-            "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, "");
+    public static final Configuration TEST_CFG = new Configuration(new String[]{"none"}, org.elasticsearch.xpack.ql.util.DateUtils.UTC,
+        "nobody", "cluster", null, TimeValue.timeValueSeconds(30), false, "");
 
     public static Configuration randomConfiguration() {
-        return new Configuration(new String[] {randomAlphaOfLength(16)},
-                randomZone(),
-                randomAlphaOfLength(16),
-                randomAlphaOfLength(16),
-                null,
-                new TimeValue(randomNonNegativeLong()),
-                randomBoolean(),
-                randomAlphaOfLength(16));
+        return new Configuration(new String[]{randomAlphaOfLength(16)},
+            randomZone(),
+            randomAlphaOfLength(16),
+            randomAlphaOfLength(16),
+            null,
+            new TimeValue(randomNonNegativeLong()),
+            randomBoolean(),
+            randomAlphaOfLength(16));
     }
 }