From 76a305375d35b98908b4c2e4bf8643896cd2d08e Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 11 Feb 2021 09:28:48 +0200 Subject: [PATCH 1/3] Add "fields" to the request and pass it way the through the query dsl --- .../client/eql/EqlSearchRequest.java | 22 ++++- .../client/eql/EqlSearchResponseTests.java | 40 +++++++- .../rest-api-spec/test/eql/10_basic.yml | 93 +++++++++++++++++++ .../xpack/eql/action/EqlSearchRequest.java | 51 +++++++++- .../xpack/eql/action/EqlSearchResponse.java | 58 +++++++++++- .../eql/execution/payload/EventPayload.java | 2 +- .../execution/search/BasicQueryClient.java | 6 ++ .../eql/execution/search/SourceGenerator.java | 10 +- .../execution/sequence/SequencePayload.java | 2 +- .../xpack/eql/plan/physical/EsQueryExec.java | 2 +- .../eql/plugin/TransportEqlSearchAction.java | 5 +- .../querydsl/container/QueryContainer.java | 2 +- .../xpack/eql/session/EqlConfiguration.java | 15 ++- .../AbstractBWCWireSerializingTestCase.java | 52 +++++++++++ .../elasticsearch/xpack/eql/EqlTestUtils.java | 4 +- .../eql/action/EqlSearchRequestTests.java | 13 ++- .../eql/action/EqlSearchResponseTests.java | 65 ++++++++++--- 17 files changed, 409 insertions(+), 33 deletions(-) create mode 100644 x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/AbstractBWCWireSerializingTestCase.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java index 9c0a7a50f78e0..ff3ec273a8630 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java @@ -15,9 +15,11 @@ import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.io.IOException; import java.util.Arrays; +import java.util.List; import java.util.Objects; public class EqlSearchRequest implements Validatable, ToXContentObject { @@ -29,6 +31,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { private String timestampField = "@timestamp"; private String eventCategoryField = "event.category"; private String resultPosition = "tail"; + private List fetchFields; private int size = 10; private int fetchSize = 1000; @@ -51,6 +54,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject { static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout"; static final String KEY_KEEP_ALIVE = "keep_alive"; static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion"; + static final String KEY_FETCH_FIELDS = "fields"; public EqlSearchRequest(String indices, String query) { indices(indices); @@ -80,6 +84,9 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par builder.field(KEY_KEEP_ALIVE, keepAlive); } builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion); + if (fetchFields != null) { + builder.field(KEY_FETCH_FIELDS, fetchFields); + } builder.endObject(); return builder; } @@ -145,6 +152,15 @@ public EqlSearchRequest resultPosition(String position) { return this; } + public List fetchFields() { + return fetchFields; + } + + public EqlSearchRequest fetchFields(List fetchFields) { + this.fetchFields = fetchFields; + return this; + } + public int size() { return this.size; } @@ -226,7 +242,8 @@ public boolean equals(Object o) { Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && Objects.equals(keepAlive, that.keepAlive) && Objects.equals(keepOnCompletion, that.keepOnCompletion) && - Objects.equals(resultPosition, that.resultPosition); + Objects.equals(resultPosition, that.resultPosition) && + Objects.equals(fetchFields, that.fetchFields); } @Override @@ -244,7 +261,8 @@ public int hashCode() { waitForCompletionTimeout, keepAlive, keepOnCompletion, - resultPosition); + resultPosition, + fetchFields); } public String[] indices() { diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java index b2a03ee2460bf..8bbaaece03e85 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java @@ -11,17 +11,23 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.client.AbstractResponseTestCase; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.RandomObjects; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Supplier; @@ -83,7 +89,15 @@ static List randomEv hits = new ArrayList<>(); for (int i = 0; i < size; i++) { BytesReference bytes = new RandomSource(() -> randomAlphaOfLength(10)).toBytes(xType); - hits.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event(String.valueOf(i), randomAlphaOfLength(10), bytes)); + Map fetchFields = new HashMap<>(); + for (int j = 0; j < randomIntBetween(0, 5); j++) { + fetchFields.put(randomAlphaOfLength(10), randomDocumentField(xType).v1()); + } + if (fetchFields.isEmpty() && randomBoolean()) { + fetchFields = null; + } + hits.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event(String.valueOf(i), randomAlphaOfLength(10), bytes, + fetchFields)); } } if (randomBoolean()) { @@ -92,6 +106,30 @@ static List randomEv return hits; } + private static Tuple randomDocumentField(XContentType xType) { + switch (randomIntBetween(0, 2)) { + case 0: + String fieldName = randomAlphaOfLengthBetween(3, 10); + Tuple, List> tuple = RandomObjects.randomStoredFieldValues(random(), xType); + DocumentField input = new DocumentField(fieldName, tuple.v1()); + DocumentField expected = new DocumentField(fieldName, tuple.v2()); + return Tuple.tuple(input, expected); + case 1: + List listValues = randomList(1, 5, () -> randomList(1, 5, ESTestCase::randomInt)); + DocumentField listField = new DocumentField(randomAlphaOfLength(5), listValues); + return Tuple.tuple(listField, listField); + case 2: + List objectValues = randomList(1, 5, () -> + Map.of(randomAlphaOfLength(5), randomInt(), + randomAlphaOfLength(5), randomBoolean(), + randomAlphaOfLength(5), randomAlphaOfLength(10))); + DocumentField objectField = new DocumentField(randomAlphaOfLength(5), objectValues); + return Tuple.tuple(objectField, objectField); + default: + throw new IllegalStateException(); + } + } + public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomEventsResponse(TotalHits totalHits, XContentType xType) { org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits hits = null; if (randomBoolean()) { diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml index 897f2c93ddb37..af648e3b9d8d3 100644 --- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml +++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml @@ -49,6 +49,52 @@ setup: - match: {hits.events.1._id: "2"} - match: {hits.events.2._id: "3"} +--- +"Execute EQL events query with fields filtering": + - do: + eql.search: + index: eql_test + body: + query: 'process where user == "SYSTEM"' + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid"] + + - match: {timed_out: false} + - match: {hits.total.value: 3} + - match: {hits.total.relation: "eq"} + - match: {hits.events.0._source.user: "SYSTEM"} + - match: {hits.events.0._id: "1"} + - match: {hits.events.0.fields.@timestamp: ["1580733296000"]} + - match: {hits.events.0.fields.id: [123]} + - match: {hits.events.0.fields.valid: [false]} + - match: {hits.events.1._id: "2"} + - match: {hits.events.1.fields.@timestamp: ["1580819696000"]} + - match: {hits.events.1.fields.id: [123]} + - match: {hits.events.1.fields.valid: [true]} + - match: {hits.events.2._id: "3"} + - match: {hits.events.2.fields.@timestamp: ["1580906096000"]} + - match: {hits.events.2.fields.id: [123]} + - match: {hits.events.2.fields.valid: [true]} + +--- +"Execute EQL events query with filter_path": + - do: + eql.search: + index: eql_test + filter_path: "hits.events._source.event.category,hits.events.fields.user,hits.events.fields.id" + body: + query: 'process where user == "SYSTEM"' + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","user"] + + - match: {hits.events.0._source.event.0.category: "process"} + - match: {hits.events.0.fields.id: [123]} + - match: {hits.events.0.fields.user: ["SYSTEM"]} + - match: {hits.events.1._source.event.0.category: "process"} + - match: {hits.events.1.fields.id: [123]} + - match: {hits.events.1.fields.user: ["SYSTEM"]} + - match: {hits.events.2._source.event.0.category: "process"} + - match: {hits.events.2.fields.id: [123]} + - match: {hits.events.2.fields.user: ["SYSTEM"]} + --- "Execute EQL sequence with string key.": - do: @@ -124,6 +170,53 @@ setup: - match: {hits.sequences.0.join_keys.0: true} - match: {hits.sequences.0.events.0._id: "2"} - match: {hits.sequences.0.events.1._id: "3"} + +--- +"Execute EQL sequence with fields filtering.": + - do: + eql.search: + index: eql_test + body: + query: 'sequence by user [process where user == "SYSTEM"] [process where true]' + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid"] + - match: {timed_out: false} + - match: {hits.total.value: 2} + - match: {hits.total.relation: "eq"} + - match: {hits.sequences.0.join_keys.0: "SYSTEM"} + - match: {hits.sequences.0.events.0._id: "1"} + - match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]} + - match: {hits.sequences.0.events.0.fields.id: [123]} + - match: {hits.sequences.0.events.0.fields.valid: [false]} + - match: {hits.sequences.0.events.1._id: "2"} + - match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]} + - match: {hits.sequences.0.events.1.fields.id: [123]} + - match: {hits.sequences.0.events.1.fields.valid: [true]} + - match: {hits.sequences.1.join_keys.0: "SYSTEM"} + - match: {hits.sequences.1.events.0._id: "2"} + - match: {hits.sequences.1.events.0.fields.@timestamp: ["1580819696000"]} + - match: {hits.sequences.1.events.0.fields.id: [123]} + - match: {hits.sequences.1.events.0.fields.valid: [true]} + - match: {hits.sequences.1.events.1._id: "3"} + - match: {hits.sequences.1.events.1.fields.@timestamp: ["1580906096000"]} + - match: {hits.sequences.1.events.1.fields.id: [123]} + - match: {hits.sequences.1.events.1.fields.valid: [true]} + +--- +"Execute EQL sequence with filter_path": + - do: + eql.search: + index: eql_test + filter_path: "hits.sequences.join_keys,hits.sequences.events.fields.valid" + body: + query: 'sequence by user [process where user == "SYSTEM"] [process where true]' + fields: ["id","valid"] + - match: {hits.sequences.0.join_keys.0: "SYSTEM"} + - match: {hits.sequences.0.events.0.fields.valid: [false]} + - match: {hits.sequences.0.events.1.fields.valid: [true]} + - match: {hits.sequences.1.join_keys.0: "SYSTEM"} + - match: {hits.sequences.1.events.0.fields.valid: [true]} + - match: {hits.sequences.1.events.1.fields.valid: [true]} + --- "Execute some EQL in async mode.": - do: diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java index 16c950b26e57e..aea0143140660 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchRequest.java @@ -17,16 +17,22 @@ import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ObjectParser.ValueType; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.common.xcontent.XContentParser.Token; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.function.Supplier; @@ -51,6 +57,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re private int fetchSize = RequestDefaults.FETCH_SIZE; private String query; private String resultPosition = "tail"; + private List fetchFields; // Async settings private TimeValue waitForCompletionTimeout = null; @@ -68,6 +75,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final String KEY_KEEP_ALIVE = "keep_alive"; static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion"; static final String KEY_RESULT_POSITION = "result_position"; + static final String KEY_FETCH_FIELDS = "fields"; static final ParseField FILTER = new ParseField(KEY_FILTER); static final ParseField TIMESTAMP_FIELD = new ParseField(KEY_TIMESTAMP_FIELD); @@ -80,6 +88,7 @@ public class EqlSearchRequest extends ActionRequest implements IndicesRequest.Re static final ParseField KEEP_ALIVE = new ParseField(KEY_KEEP_ALIVE); static final ParseField KEEP_ON_COMPLETION = new ParseField(KEY_KEEP_ON_COMPLETION); static final ParseField RESULT_POSITION = new ParseField(KEY_RESULT_POSITION); + static final ParseField FETCH_FIELDS_FIELD = SearchSourceBuilder.FETCH_FIELDS_FIELD; private static final ObjectParser PARSER = objectParser(EqlSearchRequest::new); @@ -106,6 +115,11 @@ public EqlSearchRequest(StreamInput in) throws IOException { if (in.getVersion().onOrAfter(Version.V_7_10_0)) { resultPosition = in.readString(); } + if (in.getVersion().onOrAfter(Version.V_7_12_0)) { + if (in.readBoolean()) { + fetchFields = in.readList(FieldAndFormat::new); + } + } } @Override @@ -176,6 +190,9 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws } builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion); builder.field(KEY_RESULT_POSITION, resultPosition); + if (fetchFields != null && fetchFields.isEmpty() == false) { + builder.field(KEY_FETCH_FIELDS, fetchFields); + } return builder; } @@ -201,6 +218,7 @@ protected static ObjectParser objectParser (p, c) -> TimeValue.parseTimeValue(p.text(), KEY_KEEP_ALIVE), KEEP_ALIVE, ObjectParser.ValueType.VALUE); parser.declareBoolean(EqlSearchRequest::keepOnCompletion, KEEP_ON_COMPLETION); parser.declareString(EqlSearchRequest::resultPosition, RESULT_POSITION); + parser.declareField(EqlSearchRequest::fetchFields, EqlSearchRequest::parseFetchFields, FETCH_FIELDS_FIELD, ValueType.VALUE_ARRAY); return parser; } @@ -303,6 +321,27 @@ public EqlSearchRequest resultPosition(String position) { return this; } + public List fetchFields() { + return fetchFields; + } + + public EqlSearchRequest fetchFields(List fetchFields) { + this.fetchFields = fetchFields; + return this; + } + + private static List parseFetchFields(XContentParser parser) throws IOException { + List result = new ArrayList<>(); + Token token = parser.currentToken(); + + if (token == Token.START_ARRAY) { + while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { + result.add(FieldAndFormat.fromXContent(parser)); + } + } + return result.isEmpty() ? null : result; + } + @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); @@ -324,6 +363,12 @@ public void writeTo(StreamOutput out) throws IOException { if (out.getVersion().onOrAfter(Version.V_7_10_0)) { // TODO: Remove after backport out.writeString(resultPosition); } + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { + out.writeBoolean(fetchFields != null); + if (fetchFields != null) { + out.writeList(fetchFields); + } + } } @Override @@ -346,7 +391,8 @@ public boolean equals(Object o) { Objects.equals(query, that.query) && Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) && Objects.equals(keepAlive, that.keepAlive) && - Objects.equals(resultPosition, that.resultPosition); + Objects.equals(resultPosition, that.resultPosition) && + Objects.equals(fetchFields, that.fetchFields); } @@ -364,7 +410,8 @@ public int hashCode() { query, waitForCompletionTimeout, keepAlive, - resultPosition); + resultPosition, + fetchFields); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java index ca4951208c987..682ee9d677efa 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java @@ -7,11 +7,13 @@ package org.elasticsearch.xpack.eql.action; import org.apache.lucene.search.TotalHits; +import org.elasticsearch.Version; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.common.io.stream.Writeable; @@ -31,7 +33,9 @@ import java.io.IOException; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; @@ -190,15 +194,18 @@ private static final class Fields { static final String INDEX = GetResult._INDEX; static final String ID = GetResult._ID; static final String SOURCE = SourceFieldMapper.NAME; + static final String FIELDS = "fields"; } private static final ParseField INDEX = new ParseField(Fields.INDEX); private static final ParseField ID = new ParseField(Fields.ID); private static final ParseField SOURCE = new ParseField(Fields.SOURCE); + private static final ParseField FIELDS = new ParseField(Fields.FIELDS); + @SuppressWarnings("unchecked") private static final ConstructingObjectParser PARSER = - new ConstructingObjectParser<>("eql/search_response_event", true, - args -> new Event((String) args[0], (String) args[1], (BytesReference) args[2])); + new ConstructingObjectParser<>("eql/search_response_event", true, + args -> new Event((String) args[0], (String) args[1], (BytesReference) args[2], (Map) args[3])); static { PARSER.declareString(constructorArg(), INDEX); @@ -209,22 +216,41 @@ private static final class Fields { return BytesReference.bytes(builder); } }, SOURCE); + PARSER.declareObject(optionalConstructorArg(), (p, c) -> { + Map fields = new HashMap<>(); + while (p.nextToken() != XContentParser.Token.END_OBJECT) { + DocumentField field = DocumentField.fromXContent(p); + fields.put(field.getName(), field); + } + return fields; + }, FIELDS); } private final String index; private final String id; private final BytesReference source; + private final Map fetchFields; - public Event(String index, String id, BytesReference source) { + public Event(String index, String id, BytesReference source, Map fetchFields) { this.index = index; this.id = id; this.source = source; + this.fetchFields = fetchFields; } public Event(StreamInput in) throws IOException { index = in.readString(); id = in.readString(); source = in.readBytesReference(); + if (in.getVersion().onOrAfter(Version.V_7_12_0)) { + if (in.readBoolean()) { + fetchFields = in.readMap(StreamInput::readString, DocumentField::new); + } else { + fetchFields = null; + } + } else { + fetchFields = null; + } } @Override @@ -232,6 +258,12 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(index); out.writeString(id); out.writeBytesReference(source); + if (out.getVersion().onOrAfter(Version.V_7_12_0)) { + out.writeBoolean(fetchFields != null); + if (fetchFields != null) { + out.writeMap(fetchFields, StreamOutput::writeString, (stream, documentField) -> documentField.writeTo(stream)); + } + } } @Override @@ -241,6 +273,17 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field(Fields.ID, id); // We have to use the deprecated version since we don't know the content type of the original source XContentHelper.writeRawField(Fields.SOURCE, source, builder, params); + // ignore fields all together if they are all empty + if (fetchFields != null && fetchFields.isEmpty() == false + && fetchFields.values().stream().anyMatch(df -> df.getValues().size() > 0)) { + builder.startObject(Fields.FIELDS); + for (DocumentField field : fetchFields.values()) { + if (field.getValues().size() > 0) { + field.toXContent(builder, params); + } + } + builder.endObject(); + } builder.endObject(); return builder; } @@ -261,9 +304,13 @@ public BytesReference source() { return source; } + public Map fetchFields() { + return fetchFields; + } + @Override public int hashCode() { - return Objects.hash(index, id, source); + return Objects.hash(index, id, source, fetchFields); } @Override @@ -279,7 +326,8 @@ public boolean equals(Object obj) { EqlSearchResponse.Event other = (EqlSearchResponse.Event) obj; return Objects.equals(index, other.index) && Objects.equals(id, other.id) - && Objects.equals(source, other.source); + && Objects.equals(source, other.source) + && Objects.equals(fetchFields, other.fetchFields); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java index 562ddcd297e6f..1c4f834645296 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/payload/EventPayload.java @@ -25,7 +25,7 @@ public EventPayload(SearchResponse response) { List hits = RuntimeUtils.searchHits(response); values = new ArrayList<>(hits.size()); for (SearchHit hit : hits) { - values.add(new Event(hit.getIndex(), hit.getId(), hit.getSourceRef())); + values.add(new Event(hit.getIndex(), hit.getId(), hit.getSourceRef(), hit.getFields())); } } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java index 38a2b292cb9c9..903013cfddbcb 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java @@ -19,6 +19,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.tasks.TaskCancelledException; import org.elasticsearch.xpack.eql.EqlIllegalArgumentException; import org.elasticsearch.xpack.eql.session.EqlConfiguration; @@ -44,11 +45,13 @@ public class BasicQueryClient implements QueryClient { final EqlConfiguration cfg; final Client client; final String[] indices; + final List fetchFields; public BasicQueryClient(EqlSession eqlSession) { this.cfg = eqlSession.configuration(); this.client = eqlSession.client(); this.indices = cfg.indices(); + this.fetchFields = cfg.fetchFields(); } @Override @@ -137,6 +140,9 @@ public void fetchHits(Iterable> refs, ActionListener builder.fetchField(f)); + } SearchRequest search = prepareRequest(builder, false, entry.getKey()); multiSearchBuilder.add(search); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java index 74d59279d26d5..205d886356d66 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java @@ -8,6 +8,7 @@ import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.sort.FieldSortBuilder; import org.elasticsearch.search.sort.NestedSortBuilder; import org.elasticsearch.search.sort.ScriptSortBuilder.ScriptSortType; @@ -20,6 +21,8 @@ import org.elasticsearch.xpack.ql.querydsl.container.ScriptSort; import org.elasticsearch.xpack.ql.querydsl.container.Sort; +import java.util.List; + import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.search.sort.SortBuilders.fieldSort; import static org.elasticsearch.search.sort.SortBuilders.scriptSort; @@ -28,7 +31,7 @@ public abstract class SourceGenerator { private SourceGenerator() {} - public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryBuilder filter) { + public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryBuilder filter, List fetchFields) { QueryBuilder finalQuery = null; // add the source if (container.query() != null) { @@ -61,6 +64,11 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB // disable the source, as we rely on "fields" API source.fetchSource(false); + // add the "fields" to be fetched + if (fetchFields != null) { + fetchFields.stream().forEach(f -> source.fetchField(f)); + } + if (container.limit() != null) { // add size and from source.size(container.limit().absLimit()); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java index 03e63ea44be5d..da37b97050ac2 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/sequence/SequencePayload.java @@ -28,7 +28,7 @@ class SequencePayload extends AbstractPayload { List hits = docs.get(i); List events = new ArrayList<>(hits.size()); for (SearchHit hit : hits) { - events.add(new Event(hit.getIndex(), hit.getId(), hit.getSourceRef())); + events.add(new Event(hit.getIndex(), hit.getId(), hit.getSourceRef(), hit.getFields())); } values.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Sequence(s.key().asList(), events)); } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java index 5ad4e88a22a33..f79c95ded8709 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plan/physical/EsQueryExec.java @@ -55,7 +55,7 @@ public List output() { public SearchSourceBuilder source(EqlSession session) { EqlConfiguration cfg = session.configuration(); // by default use the configuration size - return SourceGenerator.sourceBuilder(queryContainer, cfg.filter()); + return SourceGenerator.sourceBuilder(queryContainer, cfg.filter(), cfg.fetchFields()); } @Override diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java index e454b7c908c4e..e38fe2995365e 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/plugin/TransportEqlSearchAction.java @@ -21,6 +21,7 @@ import org.elasticsearch.common.time.DateUtils; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.tasks.Task; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.threadpool.ThreadPool; @@ -42,6 +43,7 @@ import java.io.IOException; import java.time.ZoneId; +import java.util.List; import java.util.Map; import static org.elasticsearch.action.ActionListener.wrap; @@ -117,6 +119,7 @@ public static void operation(PlanExecutor planExecutor, EqlSearchTask task, EqlS // TODO: these should be sent by the client ZoneId zoneId = DateUtils.of("Z"); QueryBuilder filter = request.filter(); + List fetchFields = request.fetchFields(); TimeValue timeout = TimeValue.timeValueSeconds(30); String clientId = null; @@ -128,7 +131,7 @@ public static void operation(PlanExecutor planExecutor, EqlSearchTask task, EqlS .size(request.size()) .fetchSize(request.fetchSize()); - EqlConfiguration cfg = new EqlConfiguration(request.indices(), zoneId, username, clusterName, filter, timeout, + EqlConfiguration cfg = new EqlConfiguration(request.indices(), zoneId, username, clusterName, filter, fetchFields, timeout, request.indicesOptions(), request.fetchSize(), clientId, new TaskId(nodeId, task.getId()), task); executeRequestWithRetryAttempt(clusterService, listener::onFailure, onFailure -> planExecutor.eql(cfg, request.query(), params, diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java index 31a8d98df7ecd..bb0d4b1484c81 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/querydsl/container/QueryContainer.java @@ -169,7 +169,7 @@ public boolean equals(Object obj) { public String toString() { try (XContentBuilder builder = JsonXContent.contentBuilder()) { builder.humanReadable(true).prettyPrint(); - SourceGenerator.sourceBuilder(this, null).toXContent(builder, ToXContent.EMPTY_PARAMS); + SourceGenerator.sourceBuilder(this, null, null).toXContent(builder, ToXContent.EMPTY_PARAMS); return Strings.toString(builder); } catch (IOException e) { throw new EqlIllegalArgumentException("error rendering", e); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java index cbd141ccd83c6..69f07b70a074a 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/session/EqlConfiguration.java @@ -12,10 +12,12 @@ import org.elasticsearch.common.Strings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.tasks.TaskId; import org.elasticsearch.xpack.eql.action.EqlSearchTask; import java.time.ZoneId; +import java.util.List; public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configuration { @@ -29,14 +31,17 @@ public class EqlConfiguration extends org.elasticsearch.xpack.ql.session.Configu @Nullable private final QueryBuilder filter; + @Nullable + private final List fetchFields; - public EqlConfiguration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter, TimeValue requestTimeout, - IndicesOptions indicesOptions, int fetchSize, String clientId, TaskId taskId, - EqlSearchTask task) { + public EqlConfiguration(String[] indices, ZoneId zi, String username, String clusterName, QueryBuilder filter, + List fetchFields, TimeValue requestTimeout, IndicesOptions indicesOptions, int fetchSize, + String clientId, TaskId taskId, EqlSearchTask task) { super(zi, username, clusterName); this.indices = indices; this.filter = filter; + this.fetchFields = fetchFields; this.requestTimeout = requestTimeout; this.clientId = clientId; this.indicesOptions = indicesOptions; @@ -65,6 +70,10 @@ public QueryBuilder filter() { return filter; } + public List fetchFields() { + return fetchFields; + } + public String clientId() { return clientId; } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/AbstractBWCWireSerializingTestCase.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/AbstractBWCWireSerializingTestCase.java new file mode 100644 index 0000000000000..9b36787247329 --- /dev/null +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/AbstractBWCWireSerializingTestCase.java @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.eql; + +import org.elasticsearch.Version; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.test.AbstractWireSerializingTestCase; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.Version.getDeclaredVersions; +import static org.hamcrest.Matchers.equalTo; + +public abstract class AbstractBWCWireSerializingTestCase extends AbstractWireSerializingTestCase { + + private static final List ALL_VERSIONS = Collections.unmodifiableList(getDeclaredVersions(Version.class)); + private static Version EQL_GA_VERSION = Version.V_7_10_0; + + private static List getAllBWCVersions(Version version) { + return ALL_VERSIONS.stream().filter(v -> v.onOrAfter(EQL_GA_VERSION) && v.before(version) && version.isCompatible(v)).collect( + Collectors.toList()); + } + + private static final List DEFAULT_BWC_VERSIONS = getAllBWCVersions(Version.CURRENT); + + public final void testBwcSerialization() throws IOException { + for (int runs = 0; runs < NUMBER_OF_TEST_RUNS; runs++) { + T testInstance = createTestInstance(); + for (Version bwcVersion : DEFAULT_BWC_VERSIONS) { + assertBwcSerialization(testInstance, bwcVersion); + } + } + } + + protected final void assertBwcSerialization(T testInstance, Version version) throws IOException { + T deserializedInstance = copyInstance(testInstance, version); + assertOnBWCObject(testInstance, deserializedInstance, version); + } + + protected void assertOnBWCObject(T testInstance, T bwcDeserializedObject, Version version) { + assertNotSame(version.toString(), bwcDeserializedObject, testInstance); + assertThat(version.toString(), testInstance, equalTo(bwcDeserializedObject)); + assertEquals(version.toString(), testInstance.hashCode(), bwcDeserializedObject.hashCode()); + } +} diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java index 2989b437bc45f..a50a4ca1d3411 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/EqlTestUtils.java @@ -36,7 +36,7 @@ private EqlTestUtils() { } public static final EqlConfiguration TEST_CFG = new EqlConfiguration(new String[] {"none"}, - org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, TimeValue.timeValueSeconds(30), null, + org.elasticsearch.xpack.ql.util.DateUtils.UTC, "nobody", "cluster", null, null, TimeValue.timeValueSeconds(30), null, 123, "", new TaskId("test", 123), null); public static EqlConfiguration randomConfiguration() { @@ -45,6 +45,7 @@ public static EqlConfiguration randomConfiguration() { randomAlphaOfLength(16), randomAlphaOfLength(16), null, + null, new TimeValue(randomNonNegativeLong()), randomIndicesOptions(), randomIntBetween(1, 1000), @@ -70,5 +71,4 @@ public static IndicesOptions randomIndicesOptions() { return IndicesOptions.fromOptions(randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean(), randomBoolean()); } - } diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java index 57fbefbf388a1..ccad583235755 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java @@ -15,13 +15,16 @@ import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.eql.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.function.Supplier; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -56,6 +59,13 @@ protected NamedXContentRegistry xContentRegistry() { @Override protected EqlSearchRequest createTestInstance() { try { + List randomFetchFields = new ArrayList<>(); + for (int j = 0; j < randomIntBetween(0, 5); j++) { + randomFetchFields.add(new FieldAndFormat(randomAlphaOfLength(10), randomAlphaOfLength(10))); + } + if (randomFetchFields.isEmpty()) { + randomFetchFields = null; + } QueryBuilder filter = parseFilter(defaultTestFilter); EqlSearchRequest request = new EqlSearchRequest() .indices(new String[]{defaultTestIndex}) @@ -64,7 +74,8 @@ protected EqlSearchRequest createTestInstance() { .eventCategoryField(randomAlphaOfLength(10)) .fetchSize(randomIntBetween(1, 50)) .size(randomInt(50)) - .query(randomAlphaOfLength(10)); + .query(randomAlphaOfLength(10)) + .fetchFields(randomFetchFields); return request; } catch (IOException ex) { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java index 251a31e4212d0..3798a7f8bb6e5 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchResponseTests.java @@ -8,23 +8,44 @@ import org.apache.lucene.search.TotalHits; import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.ToXContentObject; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.xpack.eql.AbstractBWCSerializationTestCase; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.RandomObjects; +import org.elasticsearch.xpack.eql.AbstractBWCWireSerializingTestCase; import org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.function.Supplier; -public class EqlSearchResponseTests extends AbstractBWCSerializationTestCase { +import static org.elasticsearch.common.xcontent.XContentHelper.toXContent; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertToXContentEquivalent; + +public class EqlSearchResponseTests extends AbstractBWCWireSerializingTestCase { + + public void testFromXContent() throws IOException { + XContentType xContentType = randomFrom(XContentType.values()).canonical(); + EqlSearchResponse response = randomEqlSearchResponse(xContentType); + boolean humanReadable = randomBoolean(); + BytesReference originalBytes = toShuffledXContent(response, xContentType, ToXContent.EMPTY_PARAMS, humanReadable); + EqlSearchResponse parsed; + try (XContentParser parser = createParser(xContentType.xContent(), originalBytes)) { + parsed = EqlSearchResponse.fromXContent(parser); + } + assertToXContentEquivalent(originalBytes, toXContent(parsed, xContentType, humanReadable), xContentType); + } private static class RandomSource implements ToXContentObject { @@ -78,7 +99,14 @@ static List randomEvents(XContentType xType) { hits = new ArrayList<>(); for (int i = 0; i < size; i++) { BytesReference bytes = new RandomSource(() -> randomAlphaOfLength(10)).toBytes(xType); - hits.add(new Event(String.valueOf(i), randomAlphaOfLength(10), bytes)); + Map fetchFields = new HashMap<>(); + for (int j = 0; j < randomIntBetween(0, 5); j++) { + fetchFields.put(randomAlphaOfLength(10), randomDocumentField(xType).v1()); + } + if (fetchFields.isEmpty() && randomBoolean()) { + fetchFields = null; + } + hits.add(new Event(String.valueOf(i), randomAlphaOfLength(10), bytes, fetchFields)); } } if (randomBoolean()) { @@ -87,9 +115,28 @@ static List randomEvents(XContentType xType) { return null; } - @Override - protected EqlSearchResponse createXContextTestInstance(XContentType xContentType) { - return randomEqlSearchResponse(xContentType); + private static Tuple randomDocumentField(XContentType xType) { + switch (randomIntBetween(0, 2)) { + case 0: + String fieldName = randomAlphaOfLengthBetween(3, 10); + Tuple, List> tuple = RandomObjects.randomStoredFieldValues(random(), xType); + DocumentField input = new DocumentField(fieldName, tuple.v1()); + DocumentField expected = new DocumentField(fieldName, tuple.v2()); + return Tuple.tuple(input, expected); + case 1: + List listValues = randomList(1, 5, () -> randomList(1, 5, ESTestCase::randomInt)); + DocumentField listField = new DocumentField(randomAlphaOfLength(5), listValues); + return Tuple.tuple(listField, listField); + case 2: + List objectValues = randomList(1, 5, () -> + Map.of(randomAlphaOfLength(5), randomInt(), + randomAlphaOfLength(5), randomBoolean(), + randomAlphaOfLength(5), randomAlphaOfLength(10))); + DocumentField objectField = new DocumentField(randomAlphaOfLength(5), objectValues); + return Tuple.tuple(objectField, objectField); + default: + throw new IllegalStateException(); + } } @Override @@ -170,9 +217,5 @@ public static EqlSearchResponse createRandomInstance(TotalHits totalHits, XConte return null; } } - - @Override - protected EqlSearchResponse doParseInstance(XContentParser parser) { - return EqlSearchResponse.fromXContent(parser); - } + } From 4c8ea55e705c8d72fa59b00e95e4e6c4d65b8684 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Sun, 14 Feb 2021 11:06:00 +0200 Subject: [PATCH 2/3] Add yaml filtering tests for runtime fields --- .../rest-api-spec/test/eql/10_basic.yml | 21 +++++++++++++++++-- .../xpack/eql/action/EqlStatusResponse.java | 2 +- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml index ef2d98719060d..13d936a1ae95b 100644 --- a/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml +++ b/x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml @@ -1,5 +1,15 @@ --- setup: + - do: + indices.create: + index: eql_test + body: + mappings: + runtime: + day_of_week: + type: keyword + script: + source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))" - do: bulk: refresh: true @@ -56,7 +66,7 @@ setup: index: eql_test body: query: 'process where user == "SYSTEM"' - fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid"] + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"] - match: {timed_out: false} - match: {hits.total.value: 3} @@ -66,14 +76,17 @@ setup: - match: {hits.events.0.fields.@timestamp: ["1580733296000"]} - match: {hits.events.0.fields.id: [123]} - match: {hits.events.0.fields.valid: [false]} + - match: {hits.events.0.fields.day_of_week: ["Monday"]} - match: {hits.events.1._id: "2"} - match: {hits.events.1.fields.@timestamp: ["1580819696000"]} - match: {hits.events.1.fields.id: [123]} - match: {hits.events.1.fields.valid: [true]} + - match: {hits.events.1.fields.day_of_week: ["Tuesday"]} - match: {hits.events.2._id: "3"} - match: {hits.events.2.fields.@timestamp: ["1580906096000"]} - match: {hits.events.2.fields.id: [123]} - match: {hits.events.2.fields.valid: [true]} + - match: {hits.events.2.fields.day_of_week: ["Wednesday"]} --- "Execute EQL events query with filter_path": @@ -178,7 +191,7 @@ setup: index: eql_test body: query: 'sequence by user [process where user == "SYSTEM"] [process where true]' - fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid"] + fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"] - match: {timed_out: false} - match: {hits.total.value: 2} - match: {hits.total.relation: "eq"} @@ -187,19 +200,23 @@ setup: - match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]} - match: {hits.sequences.0.events.0.fields.id: [123]} - match: {hits.sequences.0.events.0.fields.valid: [false]} + - match: {hits.sequences.0.events.0.fields.day_of_week: ["Monday"]} - match: {hits.sequences.0.events.1._id: "2"} - match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]} - match: {hits.sequences.0.events.1.fields.id: [123]} - match: {hits.sequences.0.events.1.fields.valid: [true]} + - match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]} - match: {hits.sequences.1.join_keys.0: "SYSTEM"} - match: {hits.sequences.1.events.0._id: "2"} - match: {hits.sequences.1.events.0.fields.@timestamp: ["1580819696000"]} - match: {hits.sequences.1.events.0.fields.id: [123]} - match: {hits.sequences.1.events.0.fields.valid: [true]} + - match: {hits.sequences.1.events.0.fields.day_of_week: ["Tuesday"]} - match: {hits.sequences.1.events.1._id: "3"} - match: {hits.sequences.1.events.1.fields.@timestamp: ["1580906096000"]} - match: {hits.sequences.1.events.1.fields.id: [123]} - match: {hits.sequences.1.events.1.fields.valid: [true]} + - match: {hits.sequences.1.events.1.fields.day_of_week: ["Wednesday"]} --- "Execute EQL sequence with filter_path": diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlStatusResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlStatusResponse.java index f78a99baa2eb0..bc1089bc2cc78 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlStatusResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlStatusResponse.java @@ -48,7 +48,7 @@ public EqlStatusResponse(String id, /** * Get status from the stored eql search response - * @param storedResponse + * @param storedResponse - stored response * @param expirationTimeMillis – expiration time in milliseconds * @param id – encoded async search id * @return a status response From 5ff728984bff7b9507494dd2123c13612838b176 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Mon, 15 Feb 2021 16:03:09 +0200 Subject: [PATCH 3/3] Address reviews --- .../xpack/eql/action/EqlSearchResponse.java | 8 ++----- .../execution/search/BasicQueryClient.java | 2 +- .../eql/execution/search/SourceGenerator.java | 2 +- .../eql/action/EqlSearchRequestTests.java | 22 ++----------------- 4 files changed, 6 insertions(+), 28 deletions(-) diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java index 682ee9d677efa..0c41429a9d2bc 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/action/EqlSearchResponse.java @@ -242,12 +242,8 @@ public Event(StreamInput in) throws IOException { index = in.readString(); id = in.readString(); source = in.readBytesReference(); - if (in.getVersion().onOrAfter(Version.V_7_12_0)) { - if (in.readBoolean()) { - fetchFields = in.readMap(StreamInput::readString, DocumentField::new); - } else { - fetchFields = null; - } + if (in.getVersion().onOrAfter(Version.V_7_12_0) && in.readBoolean()) { + fetchFields = in.readMap(StreamInput::readString, DocumentField::new); } else { fetchFields = null; } diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java index 903013cfddbcb..3caa8271f61db 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/BasicQueryClient.java @@ -141,7 +141,7 @@ public void fetchHits(Iterable> refs, ActionListener builder.fetchField(f)); + fetchFields.forEach(builder::fetchField); } SearchRequest search = prepareRequest(builder, false, entry.getKey()); diff --git a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java index 205d886356d66..1a244129487b6 100644 --- a/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java +++ b/x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/execution/search/SourceGenerator.java @@ -66,7 +66,7 @@ public static SearchSourceBuilder sourceBuilder(QueryContainer container, QueryB // add the "fields" to be fetched if (fetchFields != null) { - fetchFields.stream().forEach(f -> source.fetchField(f)); + fetchFields.forEach(source::fetchField); } if (container.limit() != null) { diff --git a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java index ccad583235755..425ff1a25e073 100644 --- a/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java +++ b/x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/action/EqlSearchRequestTests.java @@ -9,23 +9,19 @@ import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.text.Text; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.json.JsonXContent; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.search.SearchModule; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; -import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.eql.AbstractBWCSerializationTestCase; import org.junit.Before; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; -import java.util.function.Supplier; import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; @@ -60,7 +56,8 @@ protected NamedXContentRegistry xContentRegistry() { protected EqlSearchRequest createTestInstance() { try { List randomFetchFields = new ArrayList<>(); - for (int j = 0; j < randomIntBetween(0, 5); j++) { + int fetchFieldsCount = randomIntBetween(0, 5); + for (int j = 0; j < fetchFieldsCount; j++) { randomFetchFields.add(new FieldAndFormat(randomAlphaOfLength(10), randomAlphaOfLength(10))); } if (randomFetchFields.isEmpty()) { @@ -95,21 +92,6 @@ protected QueryBuilder parseFilter(XContentParser parser) throws IOException { return parseInnerQueryBuilder; } - private Object randomValue() { - Supplier value = randomFrom(Arrays.asList( - ESTestCase::randomInt, - ESTestCase::randomFloat, - ESTestCase::randomLong, - ESTestCase::randomDouble, - () -> randomAlphaOfLengthBetween(5, 20), - ESTestCase::randomBoolean, - ESTestCase::randomByte, - ESTestCase::randomShort, - () -> new Text(randomAlphaOfLengthBetween(5, 20)), - () -> null)); - return value.get(); - } - @Override protected Writeable.Reader instanceReader() { return EqlSearchRequest::new;