From 26ba4446dd9e813ea27b5cb6de22ce6294724d9b Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 21 Jun 2021 11:26:58 -0400 Subject: [PATCH] [RAC][Security Solution] Pull Gap Remediation out of search_after_bulk_create (#102104) (#102739) * Modify threshold rules to receive a single date range tuple * Modify threat match rules to receive a single date range tuple * Modify custom query rules to receive a single date range tuple * Fix up tests (partially) * Change log message to indicate single tuple instead of array * Bad test? * Prevent max_signals from being exceeded on threat match rule executions * Revert "Prevent max_signals from being exceeded on threat match rule executions" This reverts commit ba3b2f7a382ef7c369f02c7939e1495f72d92bfe. * Modify EQL rules to use date range tuple * Modify ML rules to use date range tuple * Fix ML/EQL tests * Use dateMath to parse moments in ML/Threshold tests * Add mocks for threshold test * Use dateMath for eql tests --- .../signals/executors/eql.test.ts | 10 +- .../detection_engine/signals/executors/eql.ts | 7 +- .../signals/executors/ml.test.ts | 12 +- .../detection_engine/signals/executors/ml.ts | 8 +- .../signals/executors/query.ts | 6 +- .../signals/executors/threat_match.ts | 6 +- .../signals/executors/threshold.test.ts | 25 +- .../signals/executors/threshold.ts | 160 ++++++------ .../signals/search_after_bulk_create.test.ts | 37 ++- .../signals/search_after_bulk_create.ts | 245 +++++++++--------- .../signals/signal_rule_alert_type.ts | 132 +++++----- .../threat_mapping/create_threat_signal.ts | 4 +- .../threat_mapping/create_threat_signals.ts | 4 +- .../signals/threat_mapping/types.ts | 4 +- .../lib/detection_engine/signals/types.ts | 4 +- 15 files changed, 352 insertions(+), 312 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts index 947e7d573173ea..e7af3d484dfbd2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import dateMath from '@elastic/datemath'; import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { eqlExecutor } from './eql'; @@ -23,6 +24,7 @@ describe('eql_executor', () => { let logger: ReturnType; let alertServices: AlertServicesMock; (getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION); + const params = getEqlRuleParams(); const eqlSO = { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', type: 'alert', @@ -40,10 +42,15 @@ describe('eql_executor', () => { interval: '5m', }, throttle: 'no_actions', - params: getEqlRuleParams(), + params, }, references: [], }; + const tuple = { + from: dateMath.parse(params.from)!, + to: dateMath.parse(params.to)!, + maxSignals: params.maxSignals, + }; const searchAfterSize = 7; beforeEach(() => { @@ -64,6 +71,7 @@ describe('eql_executor', () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; const response = await eqlExecutor({ rule: eqlSO, + tuple, exceptionItems, services: alertServices, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts index 28d1f3e19baeed..a187b730696829 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -28,6 +28,7 @@ import { AlertAttributes, BulkCreate, EqlSignalSearchResponse, + RuleRangeTuple, SearchAfterAndBulkCreateReturnType, WrappedSignalHit, } from '../types'; @@ -35,6 +36,7 @@ import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../uti export const eqlExecutor = async ({ rule, + tuple, exceptionItems, services, version, @@ -43,6 +45,7 @@ export const eqlExecutor = async ({ bulkCreate, }: { rule: SavedObject>; + tuple: RuleRangeTuple; exceptionItems: ExceptionListItemSchema[]; services: AlertServices; version: string; @@ -81,8 +84,8 @@ export const eqlExecutor = async ({ const request = buildEqlSearchRequest( ruleParams.query, inputIndex, - ruleParams.from, - ruleParams.to, + tuple.from.toISOString(), + tuple.to.toISOString(), searchAfterSize, ruleParams.timestampOverride, exceptionItems, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts index 25a9d2c3f510fe..89c1392cb67ba7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import dateMath from '@elastic/datemath'; import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { mlExecutor } from './ml'; @@ -26,7 +27,13 @@ describe('ml_executor', () => { const exceptionItems = [getExceptionListItemSchemaMock()]; let logger: ReturnType; let alertServices: AlertServicesMock; - const mlSO = sampleRuleSO(getMlRuleParams()); + const params = getMlRuleParams(); + const mlSO = sampleRuleSO(params); + const tuple = { + from: dateMath.parse(params.from)!, + to: dateMath.parse(params.to)!, + maxSignals: params.maxSignals, + }; const buildRuleMessage = buildRuleMessageFactory({ id: mlSO.id, ruleId: mlSO.attributes.params.ruleId, @@ -60,6 +67,7 @@ describe('ml_executor', () => { await expect( mlExecutor({ rule: mlSO, + tuple, ml: undefined, exceptionItems, services: alertServices, @@ -76,6 +84,7 @@ describe('ml_executor', () => { jobsSummaryMock.mockResolvedValue([]); const response = await mlExecutor({ rule: mlSO, + tuple, ml: mlMock, exceptionItems, services: alertServices, @@ -101,6 +110,7 @@ describe('ml_executor', () => { const response = await mlExecutor({ rule: mlSO, + tuple, ml: mlMock, exceptionItems, services: alertServices, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts index f5c7d8822b51f0..20c4cb16dadc8d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts @@ -21,11 +21,12 @@ import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; import { filterEventsAgainstList } from '../filters/filter_events_against_list'; import { findMlSignals } from '../find_ml_signals'; import { BuildRuleMessage } from '../rule_messages'; -import { AlertAttributes, BulkCreate, WrapHits } from '../types'; +import { AlertAttributes, BulkCreate, RuleRangeTuple, WrapHits } from '../types'; import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../utils'; export const mlExecutor = async ({ rule, + tuple, ml, listClient, exceptionItems, @@ -36,6 +37,7 @@ export const mlExecutor = async ({ wrapHits, }: { rule: SavedObject>; + tuple: RuleRangeTuple; ml: SetupPlugins['ml']; listClient: ListClient; exceptionItems: ExceptionListItemSchema[]; @@ -88,8 +90,8 @@ export const mlExecutor = async ({ savedObjectsClient: services.savedObjectsClient, jobIds: ruleParams.machineLearningJobId, anomalyThreshold: ruleParams.anomalyThreshold, - from: ruleParams.from, - to: ruleParams.to, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), exceptionItems, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts index 9d76a06afa2755..385c01c2f1cda1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts @@ -24,7 +24,7 @@ import { QueryRuleParams, SavedQueryRuleParams } from '../../schemas/rule_schema export const queryExecutor = async ({ rule, - tuples, + tuple, listClient, exceptionItems, services, @@ -37,7 +37,7 @@ export const queryExecutor = async ({ wrapHits, }: { rule: SavedObject>; - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; listClient: ListClient; exceptionItems: ExceptionListItemSchema[]; services: AlertServices; @@ -63,7 +63,7 @@ export const queryExecutor = async ({ }); return searchAfterAndBulkCreate({ - tuples, + tuple, listClient, exceptionsList: exceptionItems, ruleSO: rule, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts index 078eb8362069cf..d0e22f696b222e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts @@ -23,7 +23,7 @@ import { ThreatRuleParams } from '../../schemas/rule_schemas'; export const threatMatchExecutor = async ({ rule, - tuples, + tuple, listClient, exceptionItems, services, @@ -36,7 +36,7 @@ export const threatMatchExecutor = async ({ wrapHits, }: { rule: SavedObject>; - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; listClient: ListClient; exceptionItems: ExceptionListItemSchema[]; services: AlertServices; @@ -51,7 +51,7 @@ export const threatMatchExecutor = async ({ const ruleParams = rule.attributes.params; const inputIndex = await getInputIndex(services, version, ruleParams.index); return createThreatSignals({ - tuples, + tuple, threatMapping: ruleParams.threatMapping, query: ruleParams.query, inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index f03e8b8a147aea..3906c669222386 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -5,18 +5,23 @@ * 2.0. */ +import dateMath from '@elastic/datemath'; import { loggingSystemMock } from 'src/core/server/mocks'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { thresholdExecutor } from './threshold'; import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock'; import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock'; import { buildRuleMessageFactory } from '../rule_messages'; +import { sampleEmptyDocSearchResults } from '../__mocks__/es_results'; describe('threshold_executor', () => { const version = '8.0.0'; let logger: ReturnType; let alertServices: AlertServicesMock; + const params = getThresholdRuleParams(); const thresholdSO = { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', type: 'alert', @@ -34,10 +39,15 @@ describe('threshold_executor', () => { interval: '5m', }, throttle: 'no_actions', - params: getThresholdRuleParams(), + params, }, references: [], }; + const tuple = { + from: dateMath.parse(params.from)!, + to: dateMath.parse(params.to)!, + maxSignals: params.maxSignals, + }; const buildRuleMessage = buildRuleMessageFactory({ id: thresholdSO.id, ruleId: thresholdSO.attributes.params.ruleId, @@ -47,6 +57,9 @@ describe('threshold_executor', () => { beforeEach(() => { alertServices = alertsMock.createAlertServices(); + alertServices.scopedClusterClient.asCurrentUser.search.mockResolvedValue( + elasticsearchClientMock.createSuccessTransportRequestPromise(sampleEmptyDocSearchResults()) + ); logger = loggingSystemMock.createLogger(); }); @@ -55,14 +68,20 @@ describe('threshold_executor', () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; const response = await thresholdExecutor({ rule: thresholdSO, - tuples: [], + tuple, exceptionItems, services: alertServices, version, logger, buildRuleMessage, startedAt: new Date(), - bulkCreate: jest.fn(), + bulkCreate: jest.fn().mockImplementation((hits) => ({ + errors: [], + success: true, + bulkCreateDuration: '0', + createdItemsCount: 0, + createdItems: [], + })), wrapHits: jest.fn(), }); expect(response.warningMessages.length).toEqual(1); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index 5e23128c9c148a..378d68fc13d2a9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -39,7 +39,7 @@ import { BuildRuleMessage } from '../rule_messages'; export const thresholdExecutor = async ({ rule, - tuples, + tuple, exceptionItems, services, version, @@ -50,7 +50,7 @@ export const thresholdExecutor = async ({ wrapHits, }: { rule: SavedObject>; - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; exceptionItems: ExceptionListItemSchema[]; services: AlertServices; version: string; @@ -70,90 +70,88 @@ export const thresholdExecutor = async ({ } const inputIndex = await getInputIndex(services, version, ruleParams.index); - for (const tuple of tuples) { - const { - thresholdSignalHistory, - searchErrors: previousSearchErrors, - } = await getThresholdSignalHistory({ - indexPattern: [ruleParams.outputIndex], - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - ruleId: ruleParams.ruleId, - bucketByFields: ruleParams.threshold.field, - timestampOverride: ruleParams.timestampOverride, - buildRuleMessage, - }); + const { + thresholdSignalHistory, + searchErrors: previousSearchErrors, + } = await getThresholdSignalHistory({ + indexPattern: [ruleParams.outputIndex], + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + ruleId: ruleParams.ruleId, + bucketByFields: ruleParams.threshold.field, + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); - const bucketFilters = await getThresholdBucketFilters({ - thresholdSignalHistory, - timestampOverride: ruleParams.timestampOverride, - }); + const bucketFilters = await getThresholdBucketFilters({ + thresholdSignalHistory, + timestampOverride: ruleParams.timestampOverride, + }); + + const esFilter = await getFilter({ + type: ruleParams.type, + filters: ruleParams.filters ? ruleParams.filters.concat(bucketFilters) : bucketFilters, + language: ruleParams.language, + query: ruleParams.query, + savedId: ruleParams.savedId, + services, + index: inputIndex, + lists: exceptionItems, + }); + + const { + searchResult: thresholdResults, + searchErrors, + searchDuration: thresholdSearchDuration, + } = await findThresholdSignals({ + inputIndexPattern: inputIndex, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + filter: esFilter, + threshold: ruleParams.threshold, + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); - const esFilter = await getFilter({ - type: ruleParams.type, - filters: ruleParams.filters ? ruleParams.filters.concat(bucketFilters) : bucketFilters, - language: ruleParams.language, - query: ruleParams.query, - savedId: ruleParams.savedId, - services, - index: inputIndex, - lists: exceptionItems, - }); + const { + success, + bulkCreateDuration, + createdItemsCount, + createdItems, + errors, + } = await bulkCreateThresholdSignals({ + someResult: thresholdResults, + ruleSO: rule, + filter: esFilter, + services, + logger, + inputIndexPattern: inputIndex, + signalsIndex: ruleParams.outputIndex, + startedAt, + from: tuple.from.toDate(), + thresholdSignalHistory, + bulkCreate, + wrapHits, + }); - const { + result = mergeReturns([ + result, + createSearchAfterReturnTypeFromResponse({ searchResult: thresholdResults, - searchErrors, - searchDuration: thresholdSearchDuration, - } = await findThresholdSignals({ - inputIndexPattern: inputIndex, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - filter: esFilter, - threshold: ruleParams.threshold, timestampOverride: ruleParams.timestampOverride, - buildRuleMessage, - }); - - const { + }), + createSearchAfterReturnType({ success, - bulkCreateDuration, - createdItemsCount, - createdItems, - errors, - } = await bulkCreateThresholdSignals({ - someResult: thresholdResults, - ruleSO: rule, - filter: esFilter, - services, - logger, - inputIndexPattern: inputIndex, - signalsIndex: ruleParams.outputIndex, - startedAt, - from: tuple.from.toDate(), - thresholdSignalHistory, - bulkCreate, - wrapHits, - }); - - result = mergeReturns([ - result, - createSearchAfterReturnTypeFromResponse({ - searchResult: thresholdResults, - timestampOverride: ruleParams.timestampOverride, - }), - createSearchAfterReturnType({ - success, - errors: [...errors, ...previousSearchErrors, ...searchErrors], - createdSignalsCount: createdItemsCount, - createdSignals: createdItems, - bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], - searchAfterTimes: [thresholdSearchDuration], - }), - ]); - } + errors: [...errors, ...previousSearchErrors, ...searchErrors], + createdSignalsCount: createdItemsCount, + createdSignals: createdItems, + bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], + searchAfterTimes: [thresholdSearchDuration], + }), + ]); return result; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index e4eb7e854f670f..184b49c2d6c7b9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -44,14 +44,14 @@ describe('searchAfterAndBulkCreate', () => { const sampleParams = getQueryRuleParams(); const ruleSO = sampleRuleSO(getQueryRuleParams()); sampleParams.maxSignals = 30; - let tuples: RuleRangeTuple[]; + let tuple: RuleRangeTuple; beforeEach(() => { jest.clearAllMocks(); listClient = listMock.getListClient(); listClient.searchListItemByValues = jest.fn().mockResolvedValue([]); inputIndexPattern = ['auditbeat-*']; mockService = alertsMock.createAlertServices(); - ({ tuples } = getRuleRangeTuples({ + tuple = getRuleRangeTuples({ logger: mockLogger, previousStartedAt: new Date(), from: sampleParams.from, @@ -59,7 +59,7 @@ describe('searchAfterAndBulkCreate', () => { interval: '5m', maxSignals: sampleParams.maxSignals, buildRuleMessage, - })); + }).tuples[0]; bulkCreate = bulkCreateFactory( mockLogger, mockService.scopedClusterClient.asCurrentUser, @@ -174,7 +174,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - tuples, + tuple, ruleSO, listClient, exceptionsList: [exceptionItem], @@ -279,7 +279,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -357,7 +357,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -416,7 +416,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -495,7 +495,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [], services: mockService, @@ -550,7 +550,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -569,11 +569,6 @@ describe('searchAfterAndBulkCreate', () => { expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(createdSignalsCount).toEqual(0); // should not create any signals because all events were in the allowlist expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - // I don't like testing log statements since logs change but this is the best - // way I can think of to ensure this section is getting hit with this test case. - expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[7][0]).toContain( - 'ran out of sort ids to sort on name: "fake name" id: "fake id" rule id: "fake rule id" signals index: "fakeindex"' - ); }); test('should return success when no sortId present but search results are in the allowlist', async () => { @@ -627,7 +622,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -701,7 +696,7 @@ describe('searchAfterAndBulkCreate', () => { ); const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [], services: mockService, @@ -746,7 +741,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ listClient, exceptionsList: [exceptionItem], - tuples, + tuple, ruleSO, services: mockService, logger: mockLogger, @@ -793,7 +788,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ listClient, exceptionsList: [exceptionItem], - tuples, + tuple, ruleSO, services: mockService, logger: mockLogger, @@ -854,7 +849,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ listClient, exceptionsList: [exceptionItem], - tuples, + tuple, ruleSO, services: mockService, logger: mockLogger, @@ -979,7 +974,7 @@ describe('searchAfterAndBulkCreate', () => { errors, } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [], services: mockService, @@ -1075,7 +1070,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ enrichment: mockEnrichment, ruleSO, - tuples, + tuple, listClient, exceptionsList: [], services: mockService, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index bb2e57b0606e59..eb4af0c38ce254 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -23,7 +23,7 @@ import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } fr // search_after through documents and re-index using bulk endpoint. export const searchAfterAndBulkCreate = async ({ - tuples: totalToFromTuples, + tuple, ruleSO, exceptionsList, services, @@ -49,150 +49,143 @@ export const searchAfterAndBulkCreate = async ({ // to ensure we don't exceed maxSignals let signalsCreatedCount = 0; - const tuplesToBeLogged = [...totalToFromTuples]; - logger.debug(buildRuleMessage(`totalToFromTuples: ${totalToFromTuples.length}`)); - - while (totalToFromTuples.length > 0) { - const tuple = totalToFromTuples.pop(); - if (tuple == null || tuple.to == null || tuple.from == null) { - logger.error(buildRuleMessage(`[-] malformed date tuple`)); - return createSearchAfterReturnType({ - success: false, - errors: ['malformed date tuple'], - }); - } - signalsCreatedCount = 0; - while (signalsCreatedCount < tuple.maxSignals) { - try { - let mergedSearchResults = createSearchResultReturnType(); - logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); + if (tuple == null || tuple.to == null || tuple.from == null) { + logger.error(buildRuleMessage(`[-] malformed date tuple`)); + return createSearchAfterReturnType({ + success: false, + errors: ['malformed date tuple'], + }); + } + signalsCreatedCount = 0; + while (signalsCreatedCount < tuple.maxSignals) { + try { + let mergedSearchResults = createSearchResultReturnType(); + logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); - if (hasSortId) { - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ - buildRuleMessage, - searchAfterSortIds: sortIds, - index: inputIndexPattern, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - // @ts-expect-error please, declare a type explicitly instead of unknown - filter, - pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), + if (hasSortId) { + const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + buildRuleMessage, + searchAfterSortIds: sortIds, + index: inputIndexPattern, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + // @ts-expect-error please, declare a type explicitly instead of unknown + filter, + pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), + timestampOverride: ruleParams.timestampOverride, + }); + mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); + toReturn = mergeReturns([ + toReturn, + createSearchAfterReturnTypeFromResponse({ + searchResult: mergedSearchResults, timestampOverride: ruleParams.timestampOverride, - }); - mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); - toReturn = mergeReturns([ - toReturn, - createSearchAfterReturnTypeFromResponse({ - searchResult: mergedSearchResults, - timestampOverride: ruleParams.timestampOverride, - }), - createSearchAfterReturnType({ - searchAfterTimes: [searchDuration], - errors: searchErrors, - }), - ]); + }), + createSearchAfterReturnType({ + searchAfterTimes: [searchDuration], + errors: searchErrors, + }), + ]); - const lastSortIds = getSafeSortIds( - searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort - ); - if (lastSortIds != null && lastSortIds.length !== 0) { - sortIds = lastSortIds; - hasSortId = true; - } else { - hasSortId = false; - } + const lastSortIds = getSafeSortIds( + searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort + ); + if (lastSortIds != null && lastSortIds.length !== 0) { + sortIds = lastSortIds; + hasSortId = true; + } else { + hasSortId = false; } + } + + // determine if there are any candidate signals to be processed + const totalHits = createTotalHitsFromSearchResult({ searchResult: mergedSearchResults }); + logger.debug(buildRuleMessage(`totalHits: ${totalHits}`)); + logger.debug( + buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`) + ); - // determine if there are any candidate signals to be processed - const totalHits = createTotalHitsFromSearchResult({ searchResult: mergedSearchResults }); - logger.debug(buildRuleMessage(`totalHits: ${totalHits}`)); + if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { logger.debug( - buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`) + buildRuleMessage( + `${ + totalHits === 0 ? 'totalHits' : 'searchResult.hits.hits.length' + } was 0, exiting early` + ) ); + break; + } - if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { - logger.debug( - buildRuleMessage( - `${ - totalHits === 0 ? 'totalHits' : 'searchResult.hits.hits.length' - } was 0, exiting and moving on to next tuple` - ) - ); - break; - } - - // filter out the search results that match with the values found in the list. - // the resulting set are signals to be indexed, given they are not duplicates - // of signals already present in the signals index. - const filteredEvents = await filterEventsAgainstList({ - listClient, - exceptionsList, - logger, - eventSearchResult: mergedSearchResults, - buildRuleMessage, - }); - - // only bulk create if there are filteredEvents leftover - // if there isn't anything after going through the value list filter - // skip the call to bulk create and proceed to the next search_after, - // if there is a sort id to continue the search_after with. - if (filteredEvents.hits.hits.length !== 0) { - // make sure we are not going to create more signals than maxSignals allows - if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { - filteredEvents.hits.hits = filteredEvents.hits.hits.slice( - 0, - tuple.maxSignals - signalsCreatedCount - ); - } - const enrichedEvents = await enrichment(filteredEvents); - const wrappedDocs = wrapHits(enrichedEvents.hits.hits); + // filter out the search results that match with the values found in the list. + // the resulting set are signals to be indexed, given they are not duplicates + // of signals already present in the signals index. + const filteredEvents = await filterEventsAgainstList({ + listClient, + exceptionsList, + logger, + eventSearchResult: mergedSearchResults, + buildRuleMessage, + }); - const { - bulkCreateDuration: bulkDuration, - createdItemsCount: createdCount, - createdItems, - success: bulkSuccess, - errors: bulkErrors, - } = await bulkCreate(wrappedDocs); - toReturn = mergeReturns([ - toReturn, - createSearchAfterReturnType({ - success: bulkSuccess, - createdSignalsCount: createdCount, - createdSignals: createdItems, - bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, - errors: bulkErrors, - }), - ]); - signalsCreatedCount += createdCount; - logger.debug(buildRuleMessage(`created ${createdCount} signals`)); - logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); - logger.debug( - buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) + // only bulk create if there are filteredEvents leftover + // if there isn't anything after going through the value list filter + // skip the call to bulk create and proceed to the next search_after, + // if there is a sort id to continue the search_after with. + if (filteredEvents.hits.hits.length !== 0) { + // make sure we are not going to create more signals than maxSignals allows + if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { + filteredEvents.hits.hits = filteredEvents.hits.hits.slice( + 0, + tuple.maxSignals - signalsCreatedCount ); - - sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); } + const enrichedEvents = await enrichment(filteredEvents); + const wrappedDocs = wrapHits(enrichedEvents.hits.hits); - if (!hasSortId) { - logger.debug(buildRuleMessage('ran out of sort ids to sort on')); - break; - } - } catch (exc: unknown) { - logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); - return mergeReturns([ + const { + bulkCreateDuration: bulkDuration, + createdItemsCount: createdCount, + createdItems, + success: bulkSuccess, + errors: bulkErrors, + } = await bulkCreate(wrappedDocs); + toReturn = mergeReturns([ toReturn, createSearchAfterReturnType({ - success: false, - errors: [`${exc}`], + success: bulkSuccess, + createdSignalsCount: createdCount, + createdSignals: createdItems, + bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, + errors: bulkErrors, }), ]); + signalsCreatedCount += createdCount; + logger.debug(buildRuleMessage(`created ${createdCount} signals`)); + logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); + logger.debug( + buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) + ); + + sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); + } + + if (!hasSortId) { + logger.debug(buildRuleMessage('ran out of sort ids to sort on')); + break; } + } catch (exc: unknown) { + logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); + return mergeReturns([ + toReturn, + createSearchAfterReturnType({ + success: false, + errors: [`${exc}`], + }), + ]); } } logger.debug(buildRuleMessage(`[+] completed bulk index of ${toReturn.createdSignalsCount}`)); - toReturn.totalToFromTuples = tuplesToBeLogged; return toReturn; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 0a2e22bc44b60e..bb1e50c14d4014 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -235,74 +235,86 @@ export const signalRulesAlertType = ({ if (isMlRule(type)) { const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams); - result = await mlExecutor({ - rule: mlRuleSO, - ml, - listClient, - exceptionItems, - services, - logger, - buildRuleMessage, - bulkCreate, - wrapHits, - }); + for (const tuple of tuples) { + result = await mlExecutor({ + rule: mlRuleSO, + tuple, + ml, + listClient, + exceptionItems, + services, + logger, + buildRuleMessage, + bulkCreate, + wrapHits, + }); + } } else if (isThresholdRule(type)) { const thresholdRuleSO = asTypeSpecificSO(savedObject, thresholdRuleParams); - result = await thresholdExecutor({ - rule: thresholdRuleSO, - tuples, - exceptionItems, - services, - version, - logger, - buildRuleMessage, - startedAt, - bulkCreate, - wrapHits, - }); + for (const tuple of tuples) { + result = await thresholdExecutor({ + rule: thresholdRuleSO, + tuple, + exceptionItems, + services, + version, + logger, + buildRuleMessage, + startedAt, + bulkCreate, + wrapHits, + }); + } } else if (isThreatMatchRule(type)) { const threatRuleSO = asTypeSpecificSO(savedObject, threatRuleParams); - result = await threatMatchExecutor({ - rule: threatRuleSO, - tuples, - listClient, - exceptionItems, - services, - version, - searchAfterSize, - logger, - eventsTelemetry, - buildRuleMessage, - bulkCreate, - wrapHits, - }); + for (const tuple of tuples) { + result = await threatMatchExecutor({ + rule: threatRuleSO, + tuple, + listClient, + exceptionItems, + services, + version, + searchAfterSize, + logger, + eventsTelemetry, + buildRuleMessage, + bulkCreate, + wrapHits, + }); + } } else if (isQueryRule(type)) { const queryRuleSO = validateQueryRuleTypes(savedObject); - result = await queryExecutor({ - rule: queryRuleSO, - tuples, - listClient, - exceptionItems, - services, - version, - searchAfterSize, - logger, - eventsTelemetry, - buildRuleMessage, - bulkCreate, - wrapHits, - }); + for (const tuple of tuples) { + result = await queryExecutor({ + rule: queryRuleSO, + tuple, + listClient, + exceptionItems, + services, + version, + searchAfterSize, + logger, + eventsTelemetry, + buildRuleMessage, + bulkCreate, + wrapHits, + }); + } } else if (isEqlRule(type)) { const eqlRuleSO = asTypeSpecificSO(savedObject, eqlRuleParams); - result = await eqlExecutor({ - rule: eqlRuleSO, - exceptionItems, - services, - version, - searchAfterSize, - bulkCreate, - logger, - }); + for (const tuple of tuples) { + result = await eqlExecutor({ + rule: eqlRuleSO, + tuple, + exceptionItems, + services, + version, + searchAfterSize, + bulkCreate, + logger, + }); + } } else { throw new Error(`unknown rule type ${type}`); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 3e30a08f1ae69c..806f5e47608e40 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -13,7 +13,7 @@ import { CreateThreatSignalOptions } from './types'; import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ - tuples, + tuple, threatMapping, threatEnrichment, query, @@ -70,7 +70,7 @@ export const createThreatSignal = async ({ ); const result = await searchAfterAndBulkCreate({ - tuples, + tuple, listClient, exceptionsList: exceptionItems, ruleSO, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 5054ab1b2cca50..169a820392a6e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -15,7 +15,7 @@ import { combineConcurrentResults } from './utils'; import { buildThreatEnrichment } from './build_threat_enrichment'; export const createThreatSignals = async ({ - tuples, + tuple, threatMapping, query, inputIndex, @@ -104,7 +104,7 @@ export const createThreatSignals = async ({ const concurrentSearchesPerformed = chunks.map>( (slicedChunk) => createThreatSignal({ - tuples, + tuple, threatEnrichment, threatMapping, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 34b064b0f88053..ded79fc647ac41 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -40,7 +40,7 @@ import { ThreatRuleParams } from '../../schemas/rule_schemas'; export type SortOrderOrUndefined = 'asc' | 'desc' | undefined; export interface CreateThreatSignalsOptions { - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; threatMapping: ThreatMapping; query: string; inputIndex: string[]; @@ -70,7 +70,7 @@ export interface CreateThreatSignalsOptions { } export interface CreateThreatSignalOptions { - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; threatMapping: ThreatMapping; threatEnrichment: SignalsEnrichment; query: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index c35eb04ba12707..8a6ce91b2575ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -262,11 +262,11 @@ export type WrapHits = ( ) => Array>; export interface SearchAfterAndBulkCreateParams { - tuples: Array<{ + tuple: { to: moment.Moment; from: moment.Moment; maxSignals: number; - }>; + }; ruleSO: SavedObject; services: AlertServices; listClient: ListClient;