Skip to content

Commit

Permalink
[Security Solution] [Detections] Combine multiple timestamp searches …
Browse files Browse the repository at this point in the history 
…into single request (elastic#96078)

* merge multiple timestamp queries into one single search

* fix types and unit tests

* remove unused code for sending secondary search

* removes unused excludeDocsWithTimestampOverride

* adds integration tests to cover cases that should / should not generate signals when timestamp override is present in rule

* adds integration test to ensure unmapped sort fields do not break search after functionality of detection rules

* Need to figure out why moving the tests around fixed them...

* updates tests with new es archive data and fixes bug where exclusion filter was hardcoded to event.ingested :yikes:

* remove dead commented out code

* fixes typo in test file, removes redundant delete signals call in integration test, fixes logic for possibility of receving a null value in sort ids, removes unused utility function for checking valid sort ids

* a unit test for checking if an empty string of a sort id is present was failing because we moved the logic for checking that out of the build search query function and up into the big loop. So I moved that unit test into the search after bulk create test file.

* fix types

* removes isEmpty since it doesn't check for empty strings
  • Loading branch information
@dhurley14 @madirey
dhurley14 authored and madirey committed May 11, 2021
1 parent 4286384 commit 9b99d7a
Show file tree
Hide file tree
Showing 24 changed files with 653 additions and 213 deletions.
7 changes: 5 additions & 2 deletions x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/es_results.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -115,6 +115,7 @@ export const sampleDocNoSortIdNoVersion = (someUuid: string = sampleIdGuid): Sig

export const sampleDocWithSortId = (
someUuid: string = sampleIdGuid,
sortIds: string[] = ['1234567891111', '2233447556677'],
ip?: string | string[],
destIp?: string | string[]
): SignalSourceHit => ({
Expand All @@ -139,7 +140,7 @@ export const sampleDocWithSortId = (
'source.ip': ip ? (Array.isArray(ip) ? ip : [ip]) : ['127.0.0.1'],
'destination.ip': destIp ? (Array.isArray(destIp) ? destIp : [destIp]) : ['127.0.0.1'],
},
sort: ['1234567891111'],
sort: sortIds,
});

export const sampleDocNoSortId = (
Expand Down Expand Up @@ -630,7 +631,8 @@ export const repeatedSearchResultsWithSortId = (
pageSize: number,
guids: string[],
ips?: Array<string | string[]>,
destIps?: Array<string | string[]>
destIps?: Array<string | string[]>,
sortIds?: string[]
): SignalSearchResponse => ({
took: 10,
timed_out: false,
Expand All @@ -646,6 +648,7 @@ export const repeatedSearchResultsWithSortId = (
hits: Array.from({ length: pageSize }).map((x, index) => ({
...sampleDocWithSortId(
guids[index],
sortIds,
ips ? ips[index] : '127.0.0.1',
destIps ? destIps[index] : '127.0.0.1'
),
Expand Down
176 changes: 124 additions & 52 deletions .../plugins/security_solution/server/lib/detection_engine/signals/build_events_query.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -39,12 +38,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down Expand Up @@ -73,16 +79,16 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is an empty string it should not be included', () => {

test('it builds a now-5m up to today filter with timestampOverride', () => {
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
from: 'now-5m',
to: 'today',
filter: {},
size: 100,
searchAfterSortId: '',
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
searchAfterSortIds: undefined,
timestampOverride: 'event.ingested',
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -91,6 +97,10 @@ describe('create_signals', () => {
ignore_unavailable: true,
body: {
docvalue_fields: [
{
field: 'event.ingested',
format: 'strict_date_optional_time',
},
{
field: '@timestamp',
format: 'strict_date_optional_time',
Expand All @@ -104,12 +114,43 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
should: [
{
range: {
'event.ingested': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
{
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
{
bool: {
must_not: {
exists: {
field: 'event.ingested',
},
},
},
},
],
},
},
],
minimum_should_match: 1,
},
},
],
Expand All @@ -128,6 +169,12 @@ describe('create_signals', () => {
},
],
sort: [
{
'event.ingested': {
order: 'asc',
unmapped_type: 'date',
},
},
{
'@timestamp': {
order: 'asc',
Expand All @@ -138,17 +185,17 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is a valid sortId string', () => {

test('if searchAfterSortIds is a valid sortId string', () => {
const fakeSortId = '123456789012';
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
from: 'now-5m',
to: 'today',
filter: {},
size: 100,
searchAfterSortId: fakeSortId,
searchAfterSortIds: [fakeSortId],
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -170,12 +217,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down Expand Up @@ -205,17 +259,16 @@ describe('create_signals', () => {
},
});
});
test('if searchAfterSortId is a valid sortId number', () => {
test('if searchAfterSortIds is a valid sortId number', () => {
const fakeSortIdNumber = 123456789012;
const query = buildEventsSearchQuery({
index: ['auditbeat-*'],
from: 'now-5m',
to: 'today',
filter: {},
size: 100,
searchAfterSortId: fakeSortIdNumber,
searchAfterSortIds: [fakeSortIdNumber],
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -237,12 +290,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down Expand Up @@ -279,9 +339,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -303,12 +362,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down Expand Up @@ -352,9 +418,8 @@ describe('create_signals', () => {
to: 'today',
filter: {},
size: 100,
searchAfterSortId: undefined,
searchAfterSortIds: undefined,
timestampOverride: undefined,
excludeDocsWithTimestampOverride: false,
});
expect(query).toEqual({
allow_no_indices: true,
Expand All @@ -371,12 +436,19 @@ describe('create_signals', () => {
bool: {
filter: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
bool: {
minimum_should_match: 1,
should: [
{
range: {
'@timestamp': {
gte: 'now-5m',
lte: 'today',
format: 'strict_date_optional_time',
},
},
},
],
},
},
],
Expand Down
Loading

0 comments on commit 9b99d7a

Please sign in to comment.