From 78abbf695f5afc7ae3a9e912bf2113020af97285 Mon Sep 17 00:00:00 2001 From: "Devin W. Hurley" Date: Wed, 8 Apr 2020 19:58:50 -0400 Subject: [PATCH] =?UTF-8?q?[SIEM]=20[Detection=20Engine]=20Fixes=20bug=20w?= =?UTF-8?q?hen=20notification=20doesn't=E2=80=A6=20(#63013)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Set refresh on bulk create to 'wait_for' when actions are present, so we do not respond until the newly indexed signals are searchable. * set refresh on bulk create to 'wait_for' when actions are present, so we do not respond until the newly indexed signals are searchable * fix types in tests --- .../signals/bulk_create_ml_signals.ts | 3 +- .../signals/search_after_bulk_create.test.ts | 8 +++++ .../signals/search_after_bulk_create.ts | 6 +++- .../signals/signal_rule_alert_type.test.ts | 32 +++++++++++++++++++ .../signals/signal_rule_alert_type.ts | 3 ++ .../signals/single_bulk_create.test.ts | 6 ++++ .../signals/single_bulk_create.ts | 6 ++-- .../siem/server/lib/detection_engine/types.ts | 2 ++ 8 files changed, 62 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts index 355041d9efbdb..ba8938f116fc6 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -10,7 +10,7 @@ import { SearchResponse } from 'elasticsearch'; import { Logger } from '../../../../../../../../src/core/server'; import { AlertServices } from '../../../../../../../plugins/alerting/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; -import { RuleTypeParams } from '../types'; +import { RuleTypeParams, RefreshTypes } from '../types'; import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; import { AnomalyResults, Anomaly } from '../../machine_learning'; @@ -29,6 +29,7 @@ interface BulkCreateMlSignalsParams { updatedBy: string; interval: string; enabled: boolean; + refresh: RefreshTypes; tags: string[]; throttle: string; } diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 414270ffcdd5c..81600b0b8dd9b 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -52,6 +52,7 @@ describe('searchAfterAndBulkCreate', () => { enabled: true, pageSize: 1, filter: undefined, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -126,6 +127,7 @@ describe('searchAfterAndBulkCreate', () => { enabled: true, pageSize: 1, filter: undefined, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -156,6 +158,7 @@ describe('searchAfterAndBulkCreate', () => { enabled: true, pageSize: 1, filter: undefined, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -198,6 +201,7 @@ describe('searchAfterAndBulkCreate', () => { enabled: true, pageSize: 1, filter: undefined, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -240,6 +244,7 @@ describe('searchAfterAndBulkCreate', () => { enabled: true, pageSize: 1, filter: undefined, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -284,6 +289,7 @@ describe('searchAfterAndBulkCreate', () => { enabled: true, pageSize: 1, filter: undefined, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -328,6 +334,7 @@ describe('searchAfterAndBulkCreate', () => { enabled: true, pageSize: 1, filter: undefined, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -374,6 +381,7 @@ describe('searchAfterAndBulkCreate', () => { enabled: true, pageSize: 1, filter: undefined, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts index ff81730bc4a72..3a964cb91fbdb 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -6,7 +6,7 @@ import { AlertServices } from '../../../../../../../plugins/alerting/server'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; -import { RuleTypeParams } from '../types'; +import { RuleTypeParams, RefreshTypes } from '../types'; import { Logger } from '../../../../../../../../src/core/server'; import { singleSearchAfter } from './single_search_after'; import { singleBulkCreate } from './single_bulk_create'; @@ -30,6 +30,7 @@ interface SearchAfterAndBulkCreateParams { enabled: boolean; pageSize: number; filter: unknown; + refresh: RefreshTypes; tags: string[]; throttle: string; } @@ -61,6 +62,7 @@ export const searchAfterAndBulkCreate = async ({ interval, enabled, pageSize, + refresh, tags, throttle, }: SearchAfterAndBulkCreateParams): Promise => { @@ -92,6 +94,7 @@ export const searchAfterAndBulkCreate = async ({ updatedBy, interval, enabled, + refresh, tags, throttle, }); @@ -179,6 +182,7 @@ export const searchAfterAndBulkCreate = async ({ updatedBy, interval, enabled, + refresh, tags, throttle, }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 3d6f443ce60d6..03fb5832fdf42 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -105,6 +105,7 @@ describe('rules_notification_alert_type', () => { }; (ruleStatusServiceFactory as jest.Mock).mockReturnValue(ruleStatusService); (getGapBetweenRuns as jest.Mock).mockReturnValue(moment.duration(0)); + (searchAfterAndBulkCreate as jest.Mock).mockClear(); (searchAfterAndBulkCreate as jest.Mock).mockResolvedValue({ success: true, searchAfterTimes: [], @@ -149,6 +150,37 @@ describe('rules_notification_alert_type', () => { }); }); + it("should set refresh to 'wait_for' when actions are present", async () => { + const ruleAlert = getResult(); + ruleAlert.actions = [ + { + actionTypeId: '.slack', + params: { + message: + 'Rule generated {{state.signals_count}} signals\n\n{{context.rule.name}}\n{{{context.results_link}}}', + }, + group: 'default', + id: '99403909-ca9b-49ba-9d7a-7e5320e68d05', + }, + ]; + + savedObjectsClient.get.mockResolvedValue({ + id: 'id', + type: 'type', + references: [], + attributes: ruleAlert, + }); + await alert.executor(payload); + expect((searchAfterAndBulkCreate as jest.Mock).mock.calls[0][0].refresh).toEqual('wait_for'); + (searchAfterAndBulkCreate as jest.Mock).mockClear(); + }); + + it('should set refresh to false when actions are not present', async () => { + await alert.executor(payload); + expect((searchAfterAndBulkCreate as jest.Mock).mock.calls[0][0].refresh).toEqual(false); + (searchAfterAndBulkCreate as jest.Mock).mockClear(); + }); + it('should call scheduleActions if signalsCount was greater than 0 and rule has actions defined', async () => { const ruleAlert = getResult(); ruleAlert.actions = [ diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts index faac4a547fc17..0357f906f8035 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -98,6 +98,7 @@ export const signalRulesAlertType = ({ params: ruleParams, } = savedObject.attributes; const updatedAt = savedObject.updated_at ?? ''; + const refresh = actions.length ? 'wait_for' : false; const buildRuleMessage = buildRuleMessageFactory({ id: alertId, ruleId, @@ -181,6 +182,7 @@ export const signalRulesAlertType = ({ updatedAt, interval, enabled, + refresh, tags, }); result.success = success; @@ -241,6 +243,7 @@ export const signalRulesAlertType = ({ interval, enabled, pageSize: searchAfterSize, + refresh, tags, throttle, }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts index 56f061cdfa3ca..45365b446cbf0 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -159,6 +159,7 @@ describe('singleBulkCreate', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -192,6 +193,7 @@ describe('singleBulkCreate', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -217,6 +219,7 @@ describe('singleBulkCreate', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -243,6 +246,7 @@ describe('singleBulkCreate', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -271,6 +275,7 @@ describe('singleBulkCreate', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); @@ -365,6 +370,7 @@ describe('singleBulkCreate', () => { updatedBy: 'elastic', interval: '5m', enabled: true, + refresh: false, tags: ['some fake tag 1', 'some fake tag 2'], throttle: 'no_actions', }); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts index 6dd8823b57e4d..fc33d0e15e43f 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/signals/single_bulk_create.ts @@ -9,7 +9,7 @@ import { performance } from 'perf_hooks'; import { AlertServices } from '../../../../../../../plugins/alerting/server'; import { SignalSearchResponse, BulkResponse } from './types'; import { RuleAlertAction } from '../../../../common/detection_engine/types'; -import { RuleTypeParams } from '../types'; +import { RuleTypeParams, RefreshTypes } from '../types'; import { generateId, makeFloatString } from './utils'; import { buildBulkBody } from './build_bulk_body'; import { Logger } from '../../../../../../../../src/core/server'; @@ -31,6 +31,7 @@ interface SingleBulkCreateParams { enabled: boolean; tags: string[]; throttle: string; + refresh: RefreshTypes; } /** @@ -77,6 +78,7 @@ export const singleBulkCreate = async ({ updatedBy, interval, enabled, + refresh, tags, throttle, }: SingleBulkCreateParams): Promise => { @@ -124,7 +126,7 @@ export const singleBulkCreate = async ({ const start = performance.now(); const response: BulkResponse = await services.callCluster('bulk', { index: signalsIndex, - refresh: false, + refresh, body: bulkBody, }); const end = performance.now(); diff --git a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts index d3fa98fd73d3a..035f1b10ff8b2 100644 --- a/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts +++ b/x-pack/legacy/plugins/siem/server/lib/detection_engine/types.ts @@ -149,3 +149,5 @@ export type CallWithRequest, V> = ( params: T, options?: CallAPIOptions ) => Promise; + +export type RefreshTypes = false | 'wait_for';