diff --git a/docs/reference/rest-api/info.asciidoc b/docs/reference/rest-api/info.asciidoc index 7c73777dd61f7..10c0d526e189e 100644 --- a/docs/reference/rest-api/info.asciidoc +++ b/docs/reference/rest-api/info.asciidoc @@ -119,6 +119,10 @@ Example response: "available" : true, "enabled" : true }, + "eql" : { + "available" : true, + "enabled" : true + }, "sql" : { "available" : true, "enabled" : true @@ -149,6 +153,8 @@ Example response: // TESTRESPONSE[s/"expiry_date_in_millis" : 1542665112332/"expiry_date_in_millis" : "$body.license.expiry_date_in_millis"/] // TESTRESPONSE[s/"version" : "7.0.0-alpha1-SNAPSHOT",/"version": "$body.features.ml.native_code_info.version",/] // TESTRESPONSE[s/"build_hash" : "99a07c016d5a73"/"build_hash": "$body.features.ml.native_code_info.build_hash"/] +// TESTRESPONSE[s/"eql" : \{[^\}]*\},/"eql": $body.$_path,/] +// eql is disabled by default on release builds and enabled everywhere else during the initial implementation phase until its release // So much s/// but at least we test that the layout is close to matching.... The following example only returns the build and features information: diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java index 3c4c6dc0d6777..1b3fc6f44ba7e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/license/XPackLicenseState.java @@ -706,6 +706,15 @@ public boolean isEnrichAllowed() { return localStatus.active; } + /** + * Determine if EQL support should be enabled. + *

+ * EQL is available for all license types except {@link OperationMode#MISSING} + */ + public synchronized boolean isEqlAllowed() { + return status.active; + } + /** * Determine if SQL support should be enabled. *

diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index d5b312523d103..abad8feba7d42 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -37,6 +37,7 @@ import org.elasticsearch.xpack.core.beats.BeatsFeatureSetUsage; import org.elasticsearch.xpack.core.ccr.AutoFollowMetadata; import org.elasticsearch.xpack.core.deprecation.DeprecationInfoAction; +import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage; import org.elasticsearch.xpack.core.flattened.FlattenedFeatureSetUsage; import org.elasticsearch.xpack.core.frozen.FrozenIndicesFeatureSetUsage; import org.elasticsearch.xpack.core.frozen.action.FreezeIndexAction; @@ -54,9 +55,9 @@ import org.elasticsearch.xpack.core.ilm.RolloverAction; import org.elasticsearch.xpack.core.ilm.SetPriorityAction; import org.elasticsearch.xpack.core.ilm.ShrinkAction; -import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.TimeseriesLifecycleType; import org.elasticsearch.xpack.core.ilm.UnfollowAction; +import org.elasticsearch.xpack.core.ilm.WaitForSnapshotAction; import org.elasticsearch.xpack.core.ilm.action.DeleteLifecycleAction; import org.elasticsearch.xpack.core.ilm.action.ExplainLifecycleAction; import org.elasticsearch.xpack.core.ilm.action.GetLifecycleAction; @@ -500,6 +501,8 @@ public List getNamedWriteables() { new NamedWriteableRegistry.Entry(RoleMapperExpression.class, AnyExpression.NAME, AnyExpression::new), new NamedWriteableRegistry.Entry(RoleMapperExpression.class, FieldExpression.NAME, FieldExpression::new), new NamedWriteableRegistry.Entry(RoleMapperExpression.class, ExceptExpression.NAME, ExceptExpression::new), + // eql + new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.EQL, EqlFeatureSetUsage::new), // sql new NamedWriteableRegistry.Entry(XPackFeatureSet.Usage.class, XPackField.SQL, SqlFeatureSetUsage::new), // watcher diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java index 8a74272429f87..f2c682bd8a21b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackField.java @@ -27,6 +27,8 @@ public final class XPackField { public static final String UPGRADE = "upgrade"; // inside of YAML settings we still use xpack do not having handle issues with dashes public static final String SETTINGS_NAME = "xpack"; + /** Name constant for the eql feature. */ + public static final String EQL = "eql"; /** Name constant for the sql feature. */ public static final String SQL = "sql"; /** Name constant for the rollup feature. */ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java index 7a98a459f02d3..c900f353d55d1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackInfoFeatureAction.java @@ -28,6 +28,7 @@ public class XPackInfoFeatureAction extends ActionType public static final XPackInfoFeatureAction GRAPH = new XPackInfoFeatureAction(XPackField.GRAPH); public static final XPackInfoFeatureAction MACHINE_LEARNING = new XPackInfoFeatureAction(XPackField.MACHINE_LEARNING); public static final XPackInfoFeatureAction LOGSTASH = new XPackInfoFeatureAction(XPackField.LOGSTASH); + public static final XPackInfoFeatureAction EQL = new XPackInfoFeatureAction(XPackField.EQL); public static final XPackInfoFeatureAction SQL = new XPackInfoFeatureAction(XPackField.SQL); public static final XPackInfoFeatureAction ROLLUP = new XPackInfoFeatureAction(XPackField.ROLLUP); public static final XPackInfoFeatureAction INDEX_LIFECYCLE = new XPackInfoFeatureAction(XPackField.INDEX_LIFECYCLE); @@ -43,7 +44,7 @@ public class XPackInfoFeatureAction extends ActionType public static final XPackInfoFeatureAction ENRICH = new XPackInfoFeatureAction(XPackField.ENRICH); public static final List ALL = Arrays.asList( - SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, + SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS, ENRICH ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java index fe43f9661488a..ffc1651ab74ef 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/action/XPackUsageFeatureAction.java @@ -28,6 +28,7 @@ public class XPackUsageFeatureAction extends ActionType ALL = Arrays.asList( - SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, + SECURITY, MONITORING, WATCHER, GRAPH, MACHINE_LEARNING, LOGSTASH, EQL, SQL, ROLLUP, INDEX_LIFECYCLE, SNAPSHOT_LIFECYCLE, CCR, TRANSFORM, FLATTENED, VECTORS, VOTING_ONLY, FROZEN_INDICES, SPATIAL, ANALYTICS ); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java new file mode 100644 index 0000000000000..55b0ff3d051d7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/eql/EqlFeatureSetUsage.java @@ -0,0 +1,51 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.core.eql; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.XPackFeatureSet; +import org.elasticsearch.xpack.core.XPackField; + +import java.io.IOException; +import java.util.Map; + +public class EqlFeatureSetUsage extends XPackFeatureSet.Usage { + + private final Map stats; + + public EqlFeatureSetUsage(StreamInput in) throws IOException { + super(in); + stats = in.readMap(); + } + + public EqlFeatureSetUsage(boolean available, boolean enabled, Map stats) { + super(XPackField.EQL, available, enabled); + this.stats = stats; + } + + public Map stats() { + return stats; + } + + @Override + protected void innerXContent(XContentBuilder builder, Params params) throws IOException { + super.innerXContent(builder, params); + if (enabled) { + for (Map.Entry entry : stats.entrySet()) { + builder.field(entry.getKey(), entry.getValue()); + } + } + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeMap(stats); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java new file mode 100644 index 0000000000000..9fc238a8436db --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlInfoTransportAction.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql; + +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.XPackField; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureTransportAction; +import org.elasticsearch.xpack.eql.plugin.EqlPlugin; + +public class EqlInfoTransportAction extends XPackInfoFeatureTransportAction { + + private final boolean enabled; + private final XPackLicenseState licenseState; + + @Inject + public EqlInfoTransportAction(TransportService transportService, ActionFilters actionFilters, + Settings settings, XPackLicenseState licenseState) { + super(XPackInfoFeatureAction.EQL.name(), transportService, actionFilters); + this.enabled = EqlPlugin.isEnabled(settings); + this.licenseState = licenseState; + } + + @Override + public String name() { + return XPackField.EQL; + } + + @Override + public boolean available() { + return licenseState.isEqlAllowed(); + } + + @Override + public boolean enabled() { + return enabled; + } + +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java new file mode 100644 index 0000000000000..3d5b9affe9091 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/EqlUsageTransportAction.java @@ -0,0 +1,75 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.protocol.xpack.XPackUsageRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureTransportAction; +import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage; +import org.elasticsearch.xpack.core.watcher.common.stats.Counters; +import org.elasticsearch.xpack.eql.plugin.EqlPlugin; +import org.elasticsearch.xpack.eql.plugin.EqlStatsAction; +import org.elasticsearch.xpack.eql.plugin.EqlStatsRequest; +import org.elasticsearch.xpack.eql.plugin.EqlStatsResponse; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +public class EqlUsageTransportAction extends XPackUsageFeatureTransportAction { + private final boolean enabled; + private final XPackLicenseState licenseState; + private final Client client; + + @Inject + public EqlUsageTransportAction(TransportService transportService, ClusterService clusterService, ThreadPool threadPool, + ActionFilters actionFilters, IndexNameExpressionResolver indexNameExpressionResolver, + Settings settings, XPackLicenseState licenseState, Client client) { + super(XPackUsageFeatureAction.EQL.name(), transportService, clusterService, threadPool, actionFilters, + indexNameExpressionResolver); + this.enabled = EqlPlugin.isEnabled(settings); + this.licenseState = licenseState; + this.client = client; + } + + @Override + protected void masterOperation(Task task, XPackUsageRequest request, ClusterState state, + ActionListener listener) { + boolean available = licenseState.isEqlAllowed(); + if (enabled) { + EqlStatsRequest eqlRequest = new EqlStatsRequest(); + eqlRequest.includeStats(true); + eqlRequest.setParentTask(clusterService.localNode().getId(), task.getId()); + client.execute(EqlStatsAction.INSTANCE, eqlRequest, ActionListener.wrap(r -> { + List countersPerNode = r.getNodes() + .stream() + .map(EqlStatsResponse.NodeStatsResponse::getStats) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + Counters mergedCounters = Counters.merge(countersPerNode); + EqlFeatureSetUsage usage = new EqlFeatureSetUsage(available, enabled, mergedCounters.toNestedMap()); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + }, listener::onFailure)); + } else { + EqlFeatureSetUsage usage = new EqlFeatureSetUsage(available, enabled, Collections.emptyMap()); + listener.onResponse(new XPackUsageFeatureResponse(usage)); + } + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java index abed7c607acfd..cc651e7014182 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlPlugin.java @@ -28,6 +28,10 @@ import org.elasticsearch.script.ScriptService; import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.watcher.ResourceWatcherService; +import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; +import org.elasticsearch.xpack.eql.EqlInfoTransportAction; +import org.elasticsearch.xpack.eql.EqlUsageTransportAction; import org.elasticsearch.xpack.eql.action.EqlSearchAction; import org.elasticsearch.xpack.eql.execution.PlanExecutor; import org.elasticsearch.xpack.ql.index.IndexResolver; @@ -65,7 +69,10 @@ private Collection createComponents(Client client, String clusterName, N @Override public List> getActions() { return Arrays.asList( - new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class) + new ActionHandler<>(EqlSearchAction.INSTANCE, TransportEqlSearchAction.class), + new ActionHandler<>(EqlStatsAction.INSTANCE, TransportEqlStatsAction.class), + new ActionHandler<>(XPackUsageFeatureAction.EQL, EqlUsageTransportAction.class), + new ActionHandler<>(XPackInfoFeatureAction.EQL, EqlInfoTransportAction.class) ); } @@ -88,7 +95,7 @@ boolean isSnapshot() { } // TODO: this needs to be used by all plugin methods - including getActions and createComponents - private boolean isEnabled(Settings settings) { + public static boolean isEnabled(Settings settings) { return EQL_ENABLED_SETTING.get(settings); } @@ -104,6 +111,6 @@ public List getRestHandlers(Settings settings, if (isEnabled(settings) == false) { return Collections.emptyList(); } - return Arrays.asList(new RestEqlSearchAction(restController)); + return Arrays.asList(new RestEqlSearchAction(restController), new RestEqlStatsAction(restController)); } } \ No newline at end of file diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsAction.java new file mode 100644 index 0000000000000..ebd481b51c041 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsAction.java @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.action.ActionType; + +public class EqlStatsAction extends ActionType { + + public static final EqlStatsAction INSTANCE = new EqlStatsAction(); + public static final String NAME = "cluster:monitor/xpack/eql/stats/dist"; + + private EqlStatsAction() { + super(NAME, EqlStatsResponse::new); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsRequest.java new file mode 100644 index 0000000000000..6577c8ac0f404 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsRequest.java @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.action.support.nodes.BaseNodeRequest; +import org.elasticsearch.action.support.nodes.BaseNodesRequest; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; + +import java.io.IOException; + +/** + * Request to gather usage statistics + */ +public class EqlStatsRequest extends BaseNodesRequest { + + private boolean includeStats; + + public EqlStatsRequest() { + super((String[]) null); + } + + public EqlStatsRequest(StreamInput in) throws IOException { + super(in); + includeStats = in.readBoolean(); + } + + public boolean includeStats() { + return includeStats; + } + + public void includeStats(boolean includeStats) { + this.includeStats = includeStats; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(includeStats); + } + + @Override + public String toString() { + return "eql_stats"; + } + + static class NodeStatsRequest extends BaseNodeRequest { + boolean includeStats; + + NodeStatsRequest(StreamInput in) throws IOException { + super(in); + includeStats = in.readBoolean(); + } + + NodeStatsRequest(EqlStatsRequest request) { + includeStats = request.includeStats(); + } + + public boolean includeStats() { + return includeStats; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(includeStats); + } + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsResponse.java new file mode 100644 index 0000000000000..7e02611a8fd65 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/EqlStatsResponse.java @@ -0,0 +1,101 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.nodes.BaseNodeResponse; +import org.elasticsearch.action.support.nodes.BaseNodesResponse; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.xpack.core.watcher.common.stats.Counters; + +import java.io.IOException; +import java.util.List; + +public class EqlStatsResponse extends BaseNodesResponse implements ToXContentObject { + + public EqlStatsResponse(StreamInput in) throws IOException { + super(in); + } + + public EqlStatsResponse(ClusterName clusterName, List nodes, List failures) { + super(clusterName, nodes, failures); + } + + @Override + protected List readNodesFrom(StreamInput in) throws IOException { + return in.readList(NodeStatsResponse::readNodeResponse); + } + + @Override + protected void writeNodesTo(StreamOutput out, List nodes) throws IOException { + out.writeList(nodes); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startArray("stats"); + for (NodeStatsResponse node : getNodes()) { + node.toXContent(builder, params); + } + builder.endArray(); + + return builder; + } + + public static class NodeStatsResponse extends BaseNodeResponse implements ToXContentObject { + + private Counters stats; + + public NodeStatsResponse(StreamInput in) throws IOException { + super(in); + if (in.readBoolean()) { + stats = new Counters(in); + } + } + + public NodeStatsResponse(DiscoveryNode node) { + super(node); + } + + public Counters getStats() { + return stats; + } + + public void setStats(Counters stats) { + this.stats = stats; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeBoolean(stats != null); + if (stats != null) { + stats.writeTo(out); + } + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + if (stats != null && stats.hasCounters()) { + builder.field("stats", stats.toNestedMap()); + } + builder.endObject(); + return builder; + } + + static EqlStatsResponse.NodeStatsResponse readNodeResponse(StreamInput in) throws IOException { + return new EqlStatsResponse.NodeStatsResponse(in); + } + + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlStatsAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlStatsAction.java new file mode 100644 index 0000000000000..c147f95b9fee1 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/RestEqlStatsAction.java @@ -0,0 +1,34 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestActions; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestEqlStatsAction extends BaseRestHandler { + + protected RestEqlStatsAction(RestController controller) { + controller.registerHandler(GET, "/_eql/stats", this); + } + + @Override + public String getName() { + return "eql_stats"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) { + EqlStatsRequest request = new EqlStatsRequest(); + return channel -> client.execute(EqlStatsAction.INSTANCE, request, new RestActions.NodesResponseRestListener<>(channel)); + } + +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java new file mode 100644 index 0000000000000..903369d33b079 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlStatsAction.java @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql.plugin; + +import org.elasticsearch.action.FailedNodeException; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.nodes.TransportNodesAction; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; + +import java.io.IOException; +import java.util.List; + +/** + * Performs the stats operation. + */ +public class TransportEqlStatsAction extends TransportNodesAction { + + // the plan executor holds the metrics + //private final PlanExecutor planExecutor; + + @Inject + public TransportEqlStatsAction(TransportService transportService, ClusterService clusterService, + ThreadPool threadPool, ActionFilters actionFilters/* , PlanExecutor planExecutor */) { + super(EqlStatsAction.NAME, threadPool, clusterService, transportService, actionFilters, + EqlStatsRequest::new, EqlStatsRequest.NodeStatsRequest::new, ThreadPool.Names.MANAGEMENT, + EqlStatsResponse.NodeStatsResponse.class); + //this.planExecutor = planExecutor; + } + + @Override + protected EqlStatsResponse newResponse(EqlStatsRequest request, List nodes, + List failures) { + return new EqlStatsResponse(clusterService.getClusterName(), nodes, failures); + } + + @Override + protected EqlStatsRequest.NodeStatsRequest newNodeRequest(EqlStatsRequest request) { + return new EqlStatsRequest.NodeStatsRequest(request); + } + + @Override + protected EqlStatsResponse.NodeStatsResponse newNodeResponse(StreamInput in) throws IOException { + return new EqlStatsResponse.NodeStatsResponse(in); + } + + @Override + protected EqlStatsResponse.NodeStatsResponse nodeOperation(EqlStatsRequest.NodeStatsRequest request, Task task) { + EqlStatsResponse.NodeStatsResponse statsResponse = new EqlStatsResponse.NodeStatsResponse(clusterService.localNode()); + //statsResponse.setStats(planExecutor.metrics().stats()); + + return statsResponse; + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/stats/FeatureMetric.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/stats/FeatureMetric.java new file mode 100644 index 0000000000000..c9734214314aa --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/stats/FeatureMetric.java @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.stats; + +import java.util.Locale; + +public enum FeatureMetric { + SEQUENCE, + JOIN, + PIPE; + + @Override + public String toString() { + return this.name().toLowerCase(Locale.ROOT); + } +} diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/stats/Metrics.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/stats/Metrics.java new file mode 100644 index 0000000000000..1dd9f2117a8a9 --- /dev/null +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/stats/Metrics.java @@ -0,0 +1,85 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.eql.stats; + +import org.elasticsearch.common.metrics.CounterMetric; +import org.elasticsearch.xpack.core.watcher.common.stats.Counters; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Map.Entry; + +/** + * Class encapsulating the metrics collected for EQL + */ +public class Metrics { + private enum OperationType { + FAILED, TOTAL; + + @Override + public String toString() { + return this.name().toLowerCase(Locale.ROOT); + } + } + + // map that holds total/failed counters for each eql "feature" (join, pipe, sequence...) + private final Map> featuresMetrics; + protected static String FPREFIX = "features."; + + public Metrics() { + Map> fMap = new LinkedHashMap<>(); + for (FeatureMetric metric : FeatureMetric.values()) { + Map metricsMap = new LinkedHashMap<>(OperationType.values().length); + for (OperationType type : OperationType.values()) { + metricsMap.put(type, new CounterMetric()); + } + + fMap.put(metric, Collections.unmodifiableMap(metricsMap)); + } + featuresMetrics = Collections.unmodifiableMap(fMap); + } + + /** + * Increments the "total" counter for a metric + * This method should be called only once per query. + */ + public void total(FeatureMetric metric) { + inc(metric, OperationType.TOTAL); + } + + /** + * Increments the "failed" counter for a metric + */ + public void failed(FeatureMetric metric) { + inc(metric, OperationType.FAILED); + } + + private void inc(FeatureMetric metric, OperationType op) { + this.featuresMetrics.get(metric).get(op).inc(); + } + + public Counters stats() { + Counters counters = new Counters(); + + // queries metrics + for (Entry> entry : featuresMetrics.entrySet()) { + String metricName = entry.getKey().toString(); + + for (OperationType type : OperationType.values()) { + long metricCounter = entry.getValue().get(type).count(); + String operationTypeName = type.toString(); + + counters.inc(FPREFIX + metricName + "." + operationTypeName, metricCounter); + counters.inc(FPREFIX + "_all." + operationTypeName, metricCounter); + } + } + + return counters; + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java new file mode 100644 index 0000000000000..2c0f12df17ad1 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlInfoTransportActionTests.java @@ -0,0 +1,122 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +package org.elasticsearch.xpack.eql; + +import org.elasticsearch.Version; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.PlainActionFuture; +import org.elasticsearch.client.Client; +import org.elasticsearch.cluster.ClusterName; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.common.xcontent.ObjectPath; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.action.XPackUsageFeatureResponse; +import org.elasticsearch.xpack.core.eql.EqlFeatureSetUsage; +import org.elasticsearch.xpack.core.watcher.common.stats.Counters; +import org.elasticsearch.xpack.eql.plugin.EqlStatsAction; +import org.elasticsearch.xpack.eql.plugin.EqlStatsResponse; +import org.junit.Before; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.core.Is.is; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.eq; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +public class EqlInfoTransportActionTests extends ESTestCase { + + private XPackLicenseState licenseState; + private Client client; + + @Before + public void init() throws Exception { + licenseState = mock(XPackLicenseState.class); + client = mock(Client.class); + ThreadPool threadPool = mock(ThreadPool.class); + ThreadContext threadContext = new ThreadContext(Settings.EMPTY); + when(threadPool.getThreadContext()).thenReturn(threadContext); + when(client.threadPool()).thenReturn(threadPool); + } + + public void testAvailable() { + EqlInfoTransportAction featureSet = new EqlInfoTransportAction( + mock(TransportService.class), mock(ActionFilters.class), Settings.EMPTY, licenseState); + boolean available = randomBoolean(); + when(licenseState.isEqlAllowed()).thenReturn(available); + assertThat(featureSet.available(), is(available)); + } + + public void testEnabled() { + boolean enabled = randomBoolean(); + Settings.Builder settings = Settings.builder(); + settings.put("xpack.eql.enabled", enabled); + + EqlInfoTransportAction featureSet = new EqlInfoTransportAction( + mock(TransportService.class), mock(ActionFilters.class), settings.build(), licenseState); + assertThat(featureSet.enabled(), is(enabled)); + } + + @SuppressWarnings("unchecked") + public void testUsageStats() throws Exception { + doAnswer(mock -> { + ActionListener listener = + (ActionListener) mock.getArguments()[2]; + + List nodes = new ArrayList<>(); + DiscoveryNode first = new DiscoveryNode("first", buildNewFakeTransportAddress(), Version.CURRENT); + EqlStatsResponse.NodeStatsResponse firstNode = new EqlStatsResponse.NodeStatsResponse(first); + Counters firstCounters = new Counters(); + firstCounters.inc("foo.foo", 1); + firstCounters.inc("foo.bar.baz", 1); + firstNode.setStats(firstCounters); + nodes.add(firstNode); + + DiscoveryNode second = new DiscoveryNode("second", buildNewFakeTransportAddress(), Version.CURRENT); + EqlStatsResponse.NodeStatsResponse secondNode = new EqlStatsResponse.NodeStatsResponse(second); + Counters secondCounters = new Counters(); + secondCounters.inc("spam", 1); + secondCounters.inc("foo.bar.baz", 4); + secondNode.setStats(secondCounters); + nodes.add(secondNode); + + listener.onResponse(new EqlStatsResponse(new ClusterName("whatever"), nodes, Collections.emptyList())); + return null; + }).when(client).execute(eq(EqlStatsAction.INSTANCE), any(), any()); + ClusterService clusterService = mock(ClusterService.class); + final DiscoveryNode mockNode = mock(DiscoveryNode.class); + when(mockNode.getId()).thenReturn("mocknode"); + when(clusterService.localNode()).thenReturn(mockNode); + + var usageAction = new EqlUsageTransportAction(mock(TransportService.class), clusterService, null, + mock(ActionFilters.class), null, Settings.builder().put("xpack.eql.enabled", true).build(), licenseState, client); + PlainActionFuture future = new PlainActionFuture<>(); + usageAction.masterOperation(mock(Task.class), null, null, future); + EqlFeatureSetUsage eqlUsage = (EqlFeatureSetUsage) future.get().getUsage(); + + long fooBarBaz = ObjectPath.eval("foo.bar.baz", eqlUsage.stats()); + long fooFoo = ObjectPath.eval("foo.foo", eqlUsage.stats()); + long spam = ObjectPath.eval("spam", eqlUsage.stats()); + + assertThat(eqlUsage.stats().keySet(), containsInAnyOrder("foo", "spam")); + assertThat(fooBarBaz, is(5L)); + assertThat(fooFoo, is(1L)); + assertThat(spam, is(1L)); + } +}