Telemetry data initial implementation

docs/reference/rest-api/info.asciidoc

@@ -119,6 +119,10 @@ Example response:
          "available" : true,
          "enabled" : true
       },
+      "eql" : {
+         "available" : true,
+         "enabled" : true
+      },
       "sql" : {
          "available" : true,
          "enabled" : true

x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java

@@ -706,6 +706,15 @@ public boolean isEnrichAllowed() {
         return localStatus.active;
     }
 
+    /**
+     * Determine if EQL support should be enabled.
+     * <p>
+     *  EQL is available for all license types except {@link OperationMode#MISSING}
+     */
+    public synchronized boolean isEqlAllowed() {
+        return status.active;
+    }
+
     /**
      * Determine if SQL support should be enabled.
      * <p>

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java

@@ -37,6 +37,7 @@
 import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage;
 import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata;
 import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction;
+import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage;
 import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage;
 import org.elasticsearch.xpack.core.frozen.FrozenIndicesFeatureSetUsage;
 import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction;
@@ -54,9 +55,9 @@
 import org.elasticsearch.xpack.core.ilm.RolloverAction;
 import org.elasticsearch.xpack.core.ilm.SetPriorityAction;
 import org.elasticsearch.xpack.core.ilm.ShrinkAction;
-import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction;
 import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType;
 import org.elasticsearch.xpack.core.ilm.UnfollowAction;
+import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction;
 import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction;
 import org.elasticsearch.xpack.core.ilm.action.ExplainLifecycleAction;
 import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction;
@@ -500,6 +501,8 @@ public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AnyExpression.NAME, AnyExpression::new),
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, FieldExpression.NAME, FieldExpression::new),
                 new NamedWriteableRegistry.Entry(RoleMapperExpression.class, ExceptExpression.NAME, ExceptExpression::new),
+                // eql
+                new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.EQL, EqlFeatureSetUsage::new),
                 // sql
                 new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SQL, SqlFeatureSetUsage::new),
                 // watcher

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java

@@ -27,6 +27,8 @@ public final class XPackField {
     public static final String UPGRADE = "upgrade";
     // inside of YAML settings we still use xpack do not having handle issues with dashes
     public static final String SETTINGS_NAME = "xpack";
+    /** Name constant for the eql feature. */
+    public static final String EQL = "eql";
     /** Name constant for the sql feature. */
     public static final String SQL = "sql";
     /** Name constant for the rollup feature. */

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackSettings.java

@@ -7,6 +7,7 @@
 package org.elasticsearch.xpack.core;
 
 import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.Build;
 import org.elasticsearch.bootstrap.JavaVersion;
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
@@ -124,6 +125,9 @@ private XPackSettings() {
     public static final Setting<Boolean> FIPS_MODE_ENABLED =
         Setting.boolSetting("xpack.security.fips_mode.enabled", false, Property.NodeScope);
 
+    /** Setting for enabling or disabling eql. Defaults to false. */
+    public static final Setting<Boolean> EQL_ENABLED = Setting.boolSetting("xpack.eql.enabled", false, Setting.Property.NodeScope);
+
     /** Setting for enabling or disabling sql. Defaults to true. */
     public static final Setting<Boolean> SQL_ENABLED = Setting.boolSetting("xpack.sql.enabled", true, Setting.Property.NodeScope);
 
@@ -231,6 +235,9 @@ public static List<Setting<?>> getAllSettings() {
         settings.add(RESERVED_REALM_ENABLED_SETTING);
         settings.add(TOKEN_SERVICE_ENABLED_SETTING);
         settings.add(API_KEY_SERVICE_ENABLED_SETTING);
+        if (Build.CURRENT.isSnapshot()) {
+            settings.add(EQL_ENABLED);
+        }
         settings.add(SQL_ENABLED);
         settings.add(USER_SETTING);
         settings.add(ROLLUP_ENABLED);

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java

@@ -28,6 +28,7 @@ public class XPackInfoFeatureAction extends ActionType<XPackInfoFeatureResponse>
     public static final XPackInfoFeatureAction GRAPH = new XPackInfoFeatureAction(XPackField.GRAPH);
     public static final XPackInfoFeatureAction MACHINE_LEARNING = new XPackInfoFeatureAction(XPackField.MACHINE_LEARNING);
     public static final XPackInfoFeatureAction LOGSTASH = new XPackInfoFeatureAction(XPackField.LOGSTASH);
+    public static final XPackInfoFeatureAction EQL = new XPackInfoFeatureAction(XPackField.EQL);
     public static final XPackInfoFeatureAction SQL = new XPackInfoFeatureAction(XPackField.SQL);
     public static final XPackInfoFeatureAction ROLLUP = new XPackInfoFeatureAction(XPackField.ROLLUP);
     public static final XPackInfoFeatureAction INDEX_LIFECYCLE = new XPackInfoFeatureAction(XPackField.INDEX_LIFECYCLE);
@@ -43,7 +44,7 @@ public class XPackInfoFeatureAction extends ActionType<XPackInfoFeatureResponse>
     public static final XPackInfoFeatureAction ENRICH = new XPackInfoFeatureAction(XPackField.ENRICH);
 
     public static final List<XPackInfoFeatureAction> ALL = Arrays.asList(
-        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
+        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
         TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH
     );
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java

@@ -28,6 +28,7 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
     public static final XPackUsageFeatureAction GRAPH = new XPackUsageFeatureAction(XPackField.GRAPH);
     public static final XPackUsageFeatureAction MACHINE_LEARNING = new XPackUsageFeatureAction(XPackField.MACHINE_LEARNING);
     public static final XPackUsageFeatureAction LOGSTASH = new XPackUsageFeatureAction(XPackField.LOGSTASH);
+    public static final XPackUsageFeatureAction EQL = new XPackUsageFeatureAction(XPackField.EQL);
     public static final XPackUsageFeatureAction SQL = new XPackUsageFeatureAction(XPackField.SQL);
     public static final XPackUsageFeatureAction ROLLUP = new XPackUsageFeatureAction(XPackField.ROLLUP);
     public static final XPackUsageFeatureAction INDEX_LIFECYCLE = new XPackUsageFeatureAction(XPackField.INDEX_LIFECYCLE);
@@ -42,7 +43,7 @@ public class XPackUsageFeatureAction extends ActionType<XPackUsageFeatureRespons
     public static final XPackUsageFeatureAction ANALYTICS = new XPackUsageFeatureAction(XPackField.ANALYTICS);
 
     public static final List<XPackUsageFeatureAction> ALL = Arrays.asList(
-        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
+        SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR,
         TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS
     );
 

x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java

@@ -0,0 +1,51 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+package org.elasticsearch.xpack.core.eql;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.xpack.core.XPackFeatureSet;
+import org.elasticsearch.xpack.core.XPackField;
+
+import java.io.IOException;
+import java.util.Map;
+
+public class EqlFeatureSetUsage extends XPackFeatureSet.Usage {
+
+    private final Map<String, Object> stats;
+
+    public EqlFeatureSetUsage(StreamInput in) throws IOException {
+        super(in);
+        stats = in.readMap();
+    }
+
+    public EqlFeatureSetUsage(boolean available, boolean enabled, Map<String, Object> stats) {
+        super(XPackField.EQL, available, enabled);
+        this.stats = stats;
+    }
+
+    public Map<String, Object> stats() {
+        return stats;
+    }
+
+    @Override
+    protected void innerXContent(XContentBuilder builder, Params params) throws IOException {
+        super.innerXContent(builder, params);
+        if (enabled) {
+            for (Map.Entry<String, Object> entry : stats.entrySet()) {
+                builder.field(entry.getKey(), entry.getValue());
+            }
+        }
+    }
+
+    @Override
+    public void writeTo(StreamOutput out) throws IOException {
+        super.writeTo(out);
+        out.writeMap(stats);
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java

@@ -0,0 +1,46 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql;
+
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackField;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction;
+
+public class EqlInfoTransportAction extends XPackInfoFeatureTransportAction {
+
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+
+    @Inject
+    public EqlInfoTransportAction(TransportService transportService, ActionFilters actionFilters,
+                                  Settings settings, XPackLicenseState licenseState) {
+        super(XPackInfoFeatureAction.EQL.name(), transportService, actionFilters);
+        this.enabled = XPackSettings.EQL_ENABLED.get(settings);
+        this.licenseState = licenseState;
+    }
+
+    @Override
+    public String name() {
+        return XPackField.EQL;
+    }
+
+    @Override
+    public boolean available() {
+        return licenseState.isEqlAllowed();
+    }
+
+    @Override
+    public boolean enabled() {
+        return enabled;
+    }
+
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java

@@ -0,0 +1,75 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+package org.elasticsearch.xpack.eql;
+
+import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.support.ActionFilters;
+import org.elasticsearch.client.Client;
+import org.elasticsearch.cluster.ClusterState;
+import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
+import org.elasticsearch.cluster.service.ClusterService;
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.license.XPackLicenseState;
+import org.elasticsearch.protocol.xpack.XPackUsageRequest;
+import org.elasticsearch.tasks.Task;
+import org.elasticsearch.threadpool.ThreadPool;
+import org.elasticsearch.transport.TransportService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction;
+import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage;
+import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
+import org.elasticsearch.xpack.eql.plugin.EqlStatsAction;
+import org.elasticsearch.xpack.eql.plugin.EqlStatsRequest;
+import org.elasticsearch.xpack.eql.plugin.EqlStatsResponse;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+
+public class EqlUsageTransportAction extends XPackUsageFeatureTransportAction {
+    private final boolean enabled;
+    private final XPackLicenseState licenseState;
+    private final Client client;
+
+    @Inject
+    public EqlUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool,
+                                   ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver,
+                                   Settings settings, XPackLicenseState licenseState, Client client) {
+        super(XPackUsageFeatureAction.EQL.name(), transportService, clusterService, threadPool, actionFilters,
+            indexNameExpressionResolver);
+        this.enabled = XPackSettings.EQL_ENABLED.get(settings);
+        this.licenseState = licenseState;
+        this.client = client;
+    }
+
+    @Override
+    protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state,
+                                   ActionListener<XPackUsageFeatureResponse> listener) {
+        boolean available = licenseState.isEqlAllowed();
+        if (enabled) {
+            EqlStatsRequest eqlRequest = new EqlStatsRequest();
+            eqlRequest.includeStats(true);
+            eqlRequest.setParentTask(clusterService.localNode().getId(), task.getId());
+            client.execute(EqlStatsAction.INSTANCE, eqlRequest, ActionListener.wrap(r -> {
+                List<Counters> countersPerNode = r.getNodes()
+                    .stream()
+                    .map(EqlStatsResponse.NodeStatsResponse::getStats)
+                    .filter(Objects::nonNull)
+                    .collect(Collectors.toList());
+                Counters mergedCounters = Counters.merge(countersPerNode);
+                EqlFeatureSetUsage usage = new EqlFeatureSetUsage(available, enabled, mergedCounters.toNestedMap());
+                listener.onResponse(new XPackUsageFeatureResponse(usage));
+            }, listener::onFailure));
+        } else {
+            EqlFeatureSetUsage usage = new EqlFeatureSetUsage(available, enabled, Collections.emptyMap());
+            listener.onResponse(new XPackUsageFeatureResponse(usage));
+        }
+    }
+}

x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java

@@ -15,7 +15,6 @@
 import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
-import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
@@ -28,6 +27,11 @@
 import org.elasticsearch.script.ScriptService;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.watcher.ResourceWatcherService;
+import org.elasticsearch.xpack.core.XPackSettings;
+import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction;
+import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction;
+import org.elasticsearch.xpack.eql.EqlInfoTransportAction;
+import org.elasticsearch.xpack.eql.EqlUsageTransportAction;
 import org.elasticsearch.xpack.eql.action.EqlSearchAction;
 import org.elasticsearch.xpack.eql.execution.PlanExecutor;
 import org.elasticsearch.xpack.ql.index.IndexResolver;
@@ -41,11 +45,11 @@
 
 public class EqlPlugin extends Plugin implements ActionPlugin {
 
-    public static final Setting<Boolean> EQL_ENABLED_SETTING = Setting.boolSetting(
-        "xpack.eql.enabled",
-        false,
-        Setting.Property.NodeScope
-    );
+    private final boolean enabled;
+
+    public EqlPlugin(final Settings settings) {
+        this.enabled = XPackSettings.EQL_ENABLED.get(settings);
+    }
 
     @Override
     public Collection<Object> createComponents(Client client, ClusterService clusterService, ThreadPool threadPool,
@@ -65,31 +69,20 @@ private Collection<Object> createComponents(Client client, String clusterName, N
     @Override
     public List<ActionHandler<? extends ActionRequest, ? extends ActionResponse>> getActions() {
         return Arrays.asList(
-            new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class)
+            new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class),
+            new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class),
+            new ActionHandler<>(XPackUsageFeatureAction.EQL, EqlUsageTransportAction.class),
+            new ActionHandler<>(XPackInfoFeatureAction.EQL, EqlInfoTransportAction.class)
         );
     }
 
-    /**
-     * The settings defined by EQL plugin.
-     *
-     * @return the settings
-     */
-    @Override
-    public List<Setting<?>> getSettings() {
-        if (isSnapshot()) {
-            return List.of(EQL_ENABLED_SETTING);
-        } else {
-            return List.of();
-        }
-    }
-
     boolean isSnapshot() {
         return Build.CURRENT.isSnapshot();
     }
 
     // TODO: this needs to be used by all plugin methods - including getActions and createComponents
-    private boolean isEnabled(Settings settings) {
-        return EQL_ENABLED_SETTING.get(settings);
+    private boolean isEnabled() {
+        return enabled;
     }
 
     @Override
@@ -101,9 +94,9 @@ public List<RestHandler> getRestHandlers(Settings settings,
                                              IndexNameExpressionResolver indexNameExpressionResolver,
                                              Supplier<DiscoveryNodes> nodesInCluster) {
 
-        if (isEnabled(settings) == false) {
+        if (isEnabled() == false) {
             return Collections.emptyList();
         }
-        return Arrays.asList(new RestEqlSearchAction(restController));
+        return Arrays.asList(new RestEqlSearchAction(restController), new RestEqlStatsAction(restController));
     }
 }