Telemetry data initial implementation

docs/reference/rest-api/info.asciidoc

@@ -119,6 +119,10 @@ Example response:
          "available" : true,
          "enabled" : true
       },
+      "eql" : {
+         "available" : true,
+         "enabled" : true
+      },
       "sql" : {
          "available" : true,
          "enabled" : true
@@ -149,6 +153,8 @@ Example response:
 // TESTRESPONSE[s/"expiry_date_in_millis" : 1542665112332/"expiry_date_in_millis" : "$body.license.expiry_date_in_millis"/]
 // TESTRESPONSE[s/"version" : "7.0.0-alpha1-SNAPSHOT",/"version": "$body.features.ml.native_code_info.version",/]
 // TESTRESPONSE[s/"build_hash" : "99a07c016d5a73"/"build_hash": "$body.features.ml.native_code_info.build_hash"/]
+// TESTRESPONSE[s/"eql" : \{[^\}]*\},/"eql": $body.$_path,/]
+// eql is disabled by default on release builds and enabled everywhere else during the initial implementation phase until its release
 // So much s/// but at least we test that the layout is close to matching....
 
 The following example only returns the build and features information:

x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java

@@ -706,6 +706,15 @@ public boolean isEnrichAllowed() {
         return localStatus.active;
     }
 
+    /**
+     * Determine if EQL support should be enabled.
+     * <p>
+     *  EQL is available for all license types except {@link OperationMode#MISSING}
+     */
+    public synchronized boolean isEqlAllowed() {
+        return status.active;
+    }
+
     /**
      * Determine if SQL support should be enabled.
      * <p>

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -37,6 +37,7 @@
 import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
+import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage;
 import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage;
 import org.elasticsearch.xpack.core.frozen.FrozenIndicesFeatureSetUsage;
 import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction;
@@ -54,9 +55,9 @@
 import org.elasticsearch.xpack.core.ilm.RolloverAction;
 import org.elasticsearch.xpack.core.ilm.SetPriorityAction;
 import org.elasticsearch.xpack.core.ilm.ShrinkAction;
-import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction;
 import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
 import org.elasticsearch.xpack.core.ilm.UnfollowAction;
+import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction;
 import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction;
 import org.elasticsearch.xpack.core.ilm.action.ExplainLifecycleAction;
 import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction;
@@ -500,6 +501,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AnyExpression.NAME, AnyExpression::new),
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, FieldExpression.NAME, FieldExpression::new),
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, ExceptExpression.NAME, ExceptExpression::new),
+                // eql
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.EQL, EqlFeatureSetUsage::new),
                 // sql
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SQL, SqlFeatureSetUsage::new),
                 // watcher

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

@@ -27,6 +27,8 @@ public final class XPackField {
     public static final String UPGRADE = "upgrade";
     // inside of YAML settings we still use xpack do not having handle issues with dashes
     public static final String SETTINGS_NAME = "xpack";
+    /** Name constant for the eql feature. */
+    public static final String EQL = "eql";
     /** Name constant for the sql feature. */
     public static final String SQL = "sql";
     /** Name constant for the rollup feature. */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java

@@ -28,6 +28,7 @@ public class XPackInfoFeatureAction extends ActionType<XPackInfoFeatureResponse>
     public static final XPackInfoFeatureAction GRAPH = new XPackInfoFeatureAction(XPackField.GRAPH);
     public static final XPackInfoFeatureAction MACHINE_LEARNING = new XPackInfoFeatureAction(XPackField.MACHINE_LEARNING);
     public static final XPackInfoFeatureAction LOGSTASH = new XPackInfoFeatureAction(XPackField.LOGSTASH);
+    public static final XPackInfoFeatureAction EQL = new XPackInfoFeatureAction(XPackField.EQL);
     public static final XPackInfoFeatureAction SQL = new XPackInfoFeatureAction(XPackField.SQL);
     public static final XPackInfoFeatureAction ROLLUP = new XPackInfoFeatureAction(XPackField.ROLLUP);
     public static final XPackInfoFeatureAction INDEX_LIFECYCLE = new XPackInfoFeatureAction(XPackField.INDEX_LIFECYCLE);
@@ -43,7 +44,7 @@ public class XPackInfoFeatureAction extends ActionType<XPackInfoFeatureResponse>
     public static final XPackInfoFeatureAction ENRICH = new XPackInfoFeatureAction(XPackField.ENRICH);
 
     public static final List<XPackInfoFeatureAction> ALL = Arrays.asList(
-        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
+        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
         TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH
     );
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java

@@ -28,6 +28,7 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
     public static final XPackUsageFeatureAction GRAPH = new XPackUsageFeatureAction(XPackField.GRAPH);
     public static final XPackUsageFeatureAction MACHINE_LEARNING = new XPackUsageFeatureAction(XPackField.MACHINE_LEARNING);
     public static final XPackUsageFeatureAction LOGSTASH = new XPackUsageFeatureAction(XPackField.LOGSTASH);
+    public static final XPackUsageFeatureAction EQL = new XPackUsageFeatureAction(XPackField.EQL);
     public static final XPackUsageFeatureAction SQL = new XPackUsageFeatureAction(XPackField.SQL);
     public static final XPackUsageFeatureAction ROLLUP = new XPackUsageFeatureAction(XPackField.ROLLUP);
     public static final XPackUsageFeatureAction INDEX_LIFECYCLE = new XPackUsageFeatureAction(XPackField.INDEX_LIFECYCLE);
@@ -42,7 +43,7 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
     public static final XPackUsageFeatureAction ANALYTICS = new XPackUsageFeatureAction(XPackField.ANALYTICS);
 
     public static final List<XPackUsageFeatureAction> ALL = Arrays.asList(
-        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
+        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
         TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS
     );
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.eql;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class EqlFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    private final Map<String, Object> stats;
+
+    public EqlFeatureSetUsage(StreamInput in) throws IOException {
+        super(in);
+        stats = in.readMap();
+    }
+
+    public EqlFeatureSetUsage(boolean available, boolean enabled, Map<String, Object> stats) {
+        super(XPackField.EQL, available, enabled);
+        this.stats = stats;
+    }
+
+    public Map<String, Object> stats() {
+        return stats;
+    }
+
+    @Override
+    protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerXContent(builder, params);
+        if (enabled) {
+            for (Map.Entry<String, Object> entry : stats.entrySet()) {
+                builder.field(entry.getKey(), entry.getValue());
+            }
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeMap(stats);
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
+
+public class EqlInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public EqlInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                  Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.EQL.name(), transportService, actionFilters);
+        this.enabled = EqlPlugin.isEnabled(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.EQL;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isEqlAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage;
+import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
+import org.elasticsearch.xpack.eql.plugin.EqlPlugin;
+import org.elasticsearch.xpack.eql.plugin.EqlStatsAction;
+import org.elasticsearch.xpack.eql.plugin.EqlStatsRequest;
+import org.elasticsearch.xpack.eql.plugin.EqlStatsResponse;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class EqlUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+    private final Client client;
+
+    @Inject
+    public EqlUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                   ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                   Settings settings, XPackLicenseState licenseState, Client client) {
+        super(XPackUsageFeatureAction.EQL.name(), transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver);
+        this.enabled = EqlPlugin.isEnabled(settings);
+        this.licenseState = licenseState;
+        this.client = client;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        boolean available = licenseState.isEqlAllowed();
+        if (enabled) {
+            EqlStatsRequest eqlRequest = new EqlStatsRequest();
+            eqlRequest.includeStats(true);
+            eqlRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+            client.execute(EqlStatsAction.INSTANCE, eqlRequest, ActionListener.wrap(r -> {
+                List<Counters> countersPerNode = r.getNodes()
+                    .stream()
+                    .map(EqlStatsResponse.NodeStatsResponse::getStats)
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toList());
+                Counters mergedCounters = Counters.merge(countersPerNode);
+                EqlFeatureSetUsage usage = new EqlFeatureSetUsage(available, enabled, mergedCounters.toNestedMap());
+                listener.onResponse(new XPackUsageFeatureResponse(usage));
+            }, listener::onFailure));
+        } else {
+            EqlFeatureSetUsage usage = new EqlFeatureSetUsage(available, enabled, Collections.emptyMap());
+            listener.onResponse(new XPackUsageFeatureResponse(usage));
+        }
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java

@@ -28,6 +28,10 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.eql.EqlInfoTransportAction;
+import org.elasticsearch.xpack.eql.EqlUsageTransportAction;
 import org.elasticsearch.xpack.eql.action.EqlSearchAction;
 import org.elasticsearch.xpack.eql.execution.PlanExecutor;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
@@ -65,7 +69,10 @@ private Collection<Object> createComponents(Client client, String clusterName, N
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(
-            new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class)
+            new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class),
+            new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class),
+            new ActionHandler<>(XPackUsageFeatureAction.EQL, EqlUsageTransportAction.class),
+            new ActionHandler<>(XPackInfoFeatureAction.EQL, EqlInfoTransportAction.class)
         );
     }
 
@@ -88,7 +95,7 @@ boolean isSnapshot() {
     }
 
     // TODO: this needs to be used by all plugin methods - including getActions and createComponents
-    private boolean isEnabled(Settings settings) {
+    public static boolean isEnabled(Settings settings) {
         return EQL_ENABLED_SETTING.get(settings);
     }
 
@@ -104,6 +111,6 @@ public List<RestHandler> getRestHandlers(Settings settings,
         if (isEnabled(settings) == false) {
             return Collections.emptyList();
         }
-        return Arrays.asList(new RestEqlSearchAction(restController));
+        return Arrays.asList(new RestEqlSearchAction(restController), new RestEqlStatsAction(restController));
     }
 }

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsAction.java

@@ -0,0 +1,19 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.eql.plugin;
+
+import org.elasticsearch.action.ActionType;
+
+public class EqlStatsAction extends ActionType<EqlStatsResponse> {
+
+    public static final EqlStatsAction INSTANCE = new EqlStatsAction();
+    public static final String NAME = "cluster:monitor/xpack/eql/stats/dist";
+
+    private EqlStatsAction() {
+        super(NAME, EqlStatsResponse::new);
+    }
+}