Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

EQL: Adds "fields" request field to the eql request (#68962) #69447

Merged
merged 6 commits into from
Feb 24, 2021
Merged

EQL: Adds "fields" request field to the eql request (#68962) #69447

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
52ba256
EQL: Adds "fields" request field to the eql request (#68962)
astefan Feb 23, 2021
008551b
Adapt to 7.x and fix the bwc serialization tests
astefan Feb 23, 2021
4086ab1
Merge branch '7.x' of https://github.com/elastic/elasticsearch into 6…
astefan Feb 24, 2021
3d22dd9
More fixes
astefan Feb 24, 2021
60fd6fe
Address reviews
astefan Feb 24, 2021
222c064
Merge branch '7.x' into 68962_7x_backport
elasticmachine Feb 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions client/rest-high-level/src/main/java/org/elasticsearch/client/eql/EqlSearchRequest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.fetch.subphase.FieldAndFormat;

import java.io.IOException;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;

public class EqlSearchRequest implements Validatable, ToXContentObject {
Expand All @@ -29,6 +31,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
private String timestampField = "@timestamp";
private String eventCategoryField = "event.category";
private String resultPosition = "tail";
private List<FieldAndFormat> fetchFields;

private int size = 10;
private int fetchSize = 1000;
Expand All @@ -51,6 +54,7 @@ public class EqlSearchRequest implements Validatable, ToXContentObject {
static final String KEY_WAIT_FOR_COMPLETION_TIMEOUT = "wait_for_completion_timeout";
static final String KEY_KEEP_ALIVE = "keep_alive";
static final String KEY_KEEP_ON_COMPLETION = "keep_on_completion";
static final String KEY_FETCH_FIELDS = "fields";

public EqlSearchRequest(String indices, String query) {
indices(indices);
Expand Down Expand Up @@ -80,6 +84,9 @@ public XContentBuilder toXContent(XContentBuilder builder, ToXContent.Params par
builder.field(KEY_KEEP_ALIVE, keepAlive);
}
builder.field(KEY_KEEP_ON_COMPLETION, keepOnCompletion);
if (fetchFields != null) {
builder.field(KEY_FETCH_FIELDS, fetchFields);
}
builder.endObject();
return builder;
}
Expand Down Expand Up @@ -145,6 +152,15 @@ public EqlSearchRequest resultPosition(String position) {
return this;
}

public List<FieldAndFormat> fetchFields() {
return fetchFields;
}

public EqlSearchRequest fetchFields(List<FieldAndFormat> fetchFields) {
this.fetchFields = fetchFields;
return this;
}

public int size() {
return this.size;
}
Expand Down Expand Up @@ -226,7 +242,8 @@ public boolean equals(Object o) {
Objects.equals(waitForCompletionTimeout, that.waitForCompletionTimeout) &&
Objects.equals(keepAlive, that.keepAlive) &&
Objects.equals(keepOnCompletion, that.keepOnCompletion) &&
Objects.equals(resultPosition, that.resultPosition);
Objects.equals(resultPosition, that.resultPosition) &&
Objects.equals(fetchFields, that.fetchFields);
}

@Override
Expand All @@ -244,7 +261,8 @@ public int hashCode() {
waitForCompletionTimeout,
keepAlive,
keepOnCompletion,
resultPosition);
resultPosition,
fetchFields);
}

public String[] indices() {
Expand Down
39 changes: 38 additions & 1 deletion ...nt/rest-high-level/src/test/java/org/elasticsearch/client/eql/EqlSearchResponseTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,17 +11,23 @@
import org.apache.lucene.search.TotalHits;
import org.elasticsearch.client.AbstractResponseTestCase;
import org.elasticsearch.common.bytes.BytesReference;
import org.elasticsearch.common.collect.Tuple;
import org.elasticsearch.common.document.DocumentField;
import org.elasticsearch.common.xcontent.ToXContent;
import org.elasticsearch.common.xcontent.ToXContentObject;
import org.elasticsearch.common.xcontent.XContentBuilder;
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.search.lookup.SourceLookup;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.RandomObjects;

import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.function.Supplier;

Expand Down Expand Up @@ -83,7 +89,16 @@ static List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event> randomEv
hits = new ArrayList<>();
for (int i = 0; i < size; i++) {
BytesReference bytes = new RandomSource(() -> randomAlphaOfLength(10)).toBytes(xType);
hits.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event(String.valueOf(i), randomAlphaOfLength(10), bytes));
Map<String, DocumentField> fetchFields = new HashMap<>();
int fieldsCount = randomIntBetween(0, 5);
for (int j = 0; j < fieldsCount; j++) {
fetchFields.put(randomAlphaOfLength(10), randomDocumentField(xType).v1());
}
if (fetchFields.isEmpty() && randomBoolean()) {
fetchFields = null;
}
hits.add(new org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event(String.valueOf(i), randomAlphaOfLength(10), bytes,
fetchFields));
}
}
if (randomBoolean()) {
Expand All @@ -92,6 +107,28 @@ static List<org.elasticsearch.xpack.eql.action.EqlSearchResponse.Event> randomEv
return hits;
}

private static Tuple<DocumentField, DocumentField> randomDocumentField(XContentType xType) {
switch (randomIntBetween(0, 2)) {
case 0:
String fieldName = randomAlphaOfLengthBetween(3, 10);
Tuple<List<Object>, List<Object>> tuple = RandomObjects.randomStoredFieldValues(random(), xType);
DocumentField input = new DocumentField(fieldName, tuple.v1());
DocumentField expected = new DocumentField(fieldName, tuple.v2());
return Tuple.tuple(input, expected);
case 1:
List<Object> listValues = randomList(1, 5, () -> randomList(1, 5, ESTestCase::randomInt));
DocumentField listField = new DocumentField(randomAlphaOfLength(5), listValues);
return Tuple.tuple(listField, listField);
case 2:
List<Object> objectValues = randomList(1, 5,
randomFrom(Arrays.asList(() -> randomAlphaOfLength(10), ESTestCase::randomInt, ESTestCase::randomBoolean)));
DocumentField objectField = new DocumentField(randomAlphaOfLength(5), objectValues);
return Tuple.tuple(objectField, objectField);
default:
throw new IllegalStateException();
}
}

public static org.elasticsearch.xpack.eql.action.EqlSearchResponse createRandomEventsResponse(TotalHits totalHits, XContentType xType) {
org.elasticsearch.xpack.eql.action.EqlSearchResponse.Hits hits = null;
if (randomBoolean()) {
Expand Down
110 changes: 110 additions & 0 deletions x-pack/plugin/eql/qa/rest/src/yamlRestTest/resources/rest-api-spec/test/eql/10_basic.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,15 @@
---
setup:
- do:
indices.create:
index: eql_test
body:
mappings:
runtime:
day_of_week:
type: keyword
script:
source: "emit(doc['@timestamp'].value.dayOfWeekEnum.getDisplayName(TextStyle.FULL, Locale.ROOT))"
- do:
bulk:
refresh: true
Expand Down Expand Up @@ -49,6 +59,55 @@ setup:
- match: {hits.events.1._id: "2"}
- match: {hits.events.2._id: "3"}

---
"Execute EQL events query with fields filtering":
- do:
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should those have maybe the:

- skip:
      version: " - 7.12.99"

block?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess the 7.12 port will have the versions kept as in original PR?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

There won't be a 7.12 port, only 7.x.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah, right, >feature!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think the EQL yaml tests run in a mixed cluster (or any other bwc context) scenario that warrants for this skip block.

eql.search:
index: eql_test
body:
query: 'process where user == "SYSTEM"'
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]

- match: {timed_out: false}
- match: {hits.total.value: 3}
- match: {hits.total.relation: "eq"}
- match: {hits.events.0._source.user: "SYSTEM"}
- match: {hits.events.0._id: "1"}
- match: {hits.events.0.fields.@timestamp: ["1580733296000"]}
- match: {hits.events.0.fields.id: [123]}
- match: {hits.events.0.fields.valid: [false]}
- match: {hits.events.0.fields.day_of_week: ["Monday"]}
- match: {hits.events.1._id: "2"}
- match: {hits.events.1.fields.@timestamp: ["1580819696000"]}
- match: {hits.events.1.fields.id: [123]}
- match: {hits.events.1.fields.valid: [true]}
- match: {hits.events.1.fields.day_of_week: ["Tuesday"]}
- match: {hits.events.2._id: "3"}
- match: {hits.events.2.fields.@timestamp: ["1580906096000"]}
- match: {hits.events.2.fields.id: [123]}
- match: {hits.events.2.fields.valid: [true]}
- match: {hits.events.2.fields.day_of_week: ["Wednesday"]}

---
"Execute EQL events query with filter_path":
- do:
eql.search:
index: eql_test
filter_path: "hits.events._source.event.category,hits.events.fields.user,hits.events.fields.id"
body:
query: 'process where user == "SYSTEM"'
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","user"]

- match: {hits.events.0._source.event.0.category: "process"}
- match: {hits.events.0.fields.id: [123]}
- match: {hits.events.0.fields.user: ["SYSTEM"]}
- match: {hits.events.1._source.event.0.category: "process"}
- match: {hits.events.1.fields.id: [123]}
- match: {hits.events.1.fields.user: ["SYSTEM"]}
- match: {hits.events.2._source.event.0.category: "process"}
- match: {hits.events.2.fields.id: [123]}
- match: {hits.events.2.fields.user: ["SYSTEM"]}

---
"Execute EQL sequence with string key.":
- do:
Expand Down Expand Up @@ -124,6 +183,57 @@ setup:
- match: {hits.sequences.0.join_keys.0: true}
- match: {hits.sequences.0.events.0._id: "2"}
- match: {hits.sequences.0.events.1._id: "3"}

---
"Execute EQL sequence with fields filtering.":
- do:
eql.search:
index: eql_test
body:
query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
fields: [{"field":"@timestamp","format":"epoch_millis"},"id","valid","day_of_week"]
- match: {timed_out: false}
- match: {hits.total.value: 2}
- match: {hits.total.relation: "eq"}
- match: {hits.sequences.0.join_keys.0: "SYSTEM"}
- match: {hits.sequences.0.events.0._id: "1"}
- match: {hits.sequences.0.events.0.fields.@timestamp: ["1580733296000"]}
- match: {hits.sequences.0.events.0.fields.id: [123]}
- match: {hits.sequences.0.events.0.fields.valid: [false]}
- match: {hits.sequences.0.events.0.fields.day_of_week: ["Monday"]}
- match: {hits.sequences.0.events.1._id: "2"}
- match: {hits.sequences.0.events.1.fields.@timestamp: ["1580819696000"]}
- match: {hits.sequences.0.events.1.fields.id: [123]}
- match: {hits.sequences.0.events.1.fields.valid: [true]}
- match: {hits.sequences.0.events.1.fields.day_of_week: ["Tuesday"]}
- match: {hits.sequences.1.join_keys.0: "SYSTEM"}
- match: {hits.sequences.1.events.0._id: "2"}
- match: {hits.sequences.1.events.0.fields.@timestamp: ["1580819696000"]}
- match: {hits.sequences.1.events.0.fields.id: [123]}
- match: {hits.sequences.1.events.0.fields.valid: [true]}
- match: {hits.sequences.1.events.0.fields.day_of_week: ["Tuesday"]}
- match: {hits.sequences.1.events.1._id: "3"}
- match: {hits.sequences.1.events.1.fields.@timestamp: ["1580906096000"]}
- match: {hits.sequences.1.events.1.fields.id: [123]}
- match: {hits.sequences.1.events.1.fields.valid: [true]}
- match: {hits.sequences.1.events.1.fields.day_of_week: ["Wednesday"]}

---
"Execute EQL sequence with filter_path":
- do:
eql.search:
index: eql_test
filter_path: "hits.sequences.join_keys,hits.sequences.events.fields.valid"
body:
query: 'sequence by user [process where user == "SYSTEM"] [process where true]'
fields: ["id","valid"]
- match: {hits.sequences.0.join_keys.0: "SYSTEM"}
- match: {hits.sequences.0.events.0.fields.valid: [false]}
- match: {hits.sequences.0.events.1.fields.valid: [true]}
- match: {hits.sequences.1.join_keys.0: "SYSTEM"}
- match: {hits.sequences.1.events.0.fields.valid: [true]}
- match: {hits.sequences.1.events.1.fields.valid: [true]}

---
"Execute some EQL in async mode.":
- do:
Expand Down
Loading