From c61395ba618b13aedf514b253812d1e4e1dcb66a Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 8 Jun 2021 21:50:23 -0400 Subject: [PATCH 01/14] Modify threshold rules to receive a single date range tuple --- .../signals/executors/threshold.ts | 160 +++++++++--------- .../signals/signal_rule_alert_type.ts | 26 +-- 2 files changed, 93 insertions(+), 93 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index 5e23128c9c148a..378d68fc13d2a9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -39,7 +39,7 @@ import { BuildRuleMessage } from '../rule_messages'; export const thresholdExecutor = async ({ rule, - tuples, + tuple, exceptionItems, services, version, @@ -50,7 +50,7 @@ export const thresholdExecutor = async ({ wrapHits, }: { rule: SavedObject>; - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; exceptionItems: ExceptionListItemSchema[]; services: AlertServices; version: string; @@ -70,90 +70,88 @@ export const thresholdExecutor = async ({ } const inputIndex = await getInputIndex(services, version, ruleParams.index); - for (const tuple of tuples) { - const { - thresholdSignalHistory, - searchErrors: previousSearchErrors, - } = await getThresholdSignalHistory({ - indexPattern: [ruleParams.outputIndex], - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - ruleId: ruleParams.ruleId, - bucketByFields: ruleParams.threshold.field, - timestampOverride: ruleParams.timestampOverride, - buildRuleMessage, - }); + const { + thresholdSignalHistory, + searchErrors: previousSearchErrors, + } = await getThresholdSignalHistory({ + indexPattern: [ruleParams.outputIndex], + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + ruleId: ruleParams.ruleId, + bucketByFields: ruleParams.threshold.field, + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); - const bucketFilters = await getThresholdBucketFilters({ - thresholdSignalHistory, - timestampOverride: ruleParams.timestampOverride, - }); + const bucketFilters = await getThresholdBucketFilters({ + thresholdSignalHistory, + timestampOverride: ruleParams.timestampOverride, + }); + + const esFilter = await getFilter({ + type: ruleParams.type, + filters: ruleParams.filters ? ruleParams.filters.concat(bucketFilters) : bucketFilters, + language: ruleParams.language, + query: ruleParams.query, + savedId: ruleParams.savedId, + services, + index: inputIndex, + lists: exceptionItems, + }); + + const { + searchResult: thresholdResults, + searchErrors, + searchDuration: thresholdSearchDuration, + } = await findThresholdSignals({ + inputIndexPattern: inputIndex, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + filter: esFilter, + threshold: ruleParams.threshold, + timestampOverride: ruleParams.timestampOverride, + buildRuleMessage, + }); - const esFilter = await getFilter({ - type: ruleParams.type, - filters: ruleParams.filters ? ruleParams.filters.concat(bucketFilters) : bucketFilters, - language: ruleParams.language, - query: ruleParams.query, - savedId: ruleParams.savedId, - services, - index: inputIndex, - lists: exceptionItems, - }); + const { + success, + bulkCreateDuration, + createdItemsCount, + createdItems, + errors, + } = await bulkCreateThresholdSignals({ + someResult: thresholdResults, + ruleSO: rule, + filter: esFilter, + services, + logger, + inputIndexPattern: inputIndex, + signalsIndex: ruleParams.outputIndex, + startedAt, + from: tuple.from.toDate(), + thresholdSignalHistory, + bulkCreate, + wrapHits, + }); - const { + result = mergeReturns([ + result, + createSearchAfterReturnTypeFromResponse({ searchResult: thresholdResults, - searchErrors, - searchDuration: thresholdSearchDuration, - } = await findThresholdSignals({ - inputIndexPattern: inputIndex, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - filter: esFilter, - threshold: ruleParams.threshold, timestampOverride: ruleParams.timestampOverride, - buildRuleMessage, - }); - - const { + }), + createSearchAfterReturnType({ success, - bulkCreateDuration, - createdItemsCount, - createdItems, - errors, - } = await bulkCreateThresholdSignals({ - someResult: thresholdResults, - ruleSO: rule, - filter: esFilter, - services, - logger, - inputIndexPattern: inputIndex, - signalsIndex: ruleParams.outputIndex, - startedAt, - from: tuple.from.toDate(), - thresholdSignalHistory, - bulkCreate, - wrapHits, - }); - - result = mergeReturns([ - result, - createSearchAfterReturnTypeFromResponse({ - searchResult: thresholdResults, - timestampOverride: ruleParams.timestampOverride, - }), - createSearchAfterReturnType({ - success, - errors: [...errors, ...previousSearchErrors, ...searchErrors], - createdSignalsCount: createdItemsCount, - createdSignals: createdItems, - bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], - searchAfterTimes: [thresholdSearchDuration], - }), - ]); - } + errors: [...errors, ...previousSearchErrors, ...searchErrors], + createdSignalsCount: createdItemsCount, + createdSignals: createdItems, + bulkCreateTimes: bulkCreateDuration ? [bulkCreateDuration] : [], + searchAfterTimes: [thresholdSearchDuration], + }), + ]); return result; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 0a2e22bc44b60e..288406ff32481a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -248,18 +248,20 @@ export const signalRulesAlertType = ({ }); } else if (isThresholdRule(type)) { const thresholdRuleSO = asTypeSpecificSO(savedObject, thresholdRuleParams); - result = await thresholdExecutor({ - rule: thresholdRuleSO, - tuples, - exceptionItems, - services, - version, - logger, - buildRuleMessage, - startedAt, - bulkCreate, - wrapHits, - }); + for (const tuple of tuples) { + result = await thresholdExecutor({ + rule: thresholdRuleSO, + tuple, + exceptionItems, + services, + version, + logger, + buildRuleMessage, + startedAt, + bulkCreate, + wrapHits, + }); + } } else if (isThreatMatchRule(type)) { const threatRuleSO = asTypeSpecificSO(savedObject, threatRuleParams); result = await threatMatchExecutor({ From 74ac5eb320cae5d7475d4d72fc1d5e798fa2556b Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 8 Jun 2021 21:57:14 -0400 Subject: [PATCH 02/14] Modify threat match rules to receive a single date range tuple --- .../signals/executors/threat_match.ts | 6 +- .../signals/search_after_bulk_create.ts | 245 +++++++++--------- .../signals/signal_rule_alert_type.ts | 30 ++- .../threat_mapping/create_threat_signal.ts | 4 +- .../threat_mapping/create_threat_signals.ts | 4 +- .../signals/threat_mapping/types.ts | 4 +- .../lib/detection_engine/signals/types.ts | 4 +- 7 files changed, 146 insertions(+), 151 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts index 078eb8362069cf..d0e22f696b222e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts @@ -23,7 +23,7 @@ import { ThreatRuleParams } from '../../schemas/rule_schemas'; export const threatMatchExecutor = async ({ rule, - tuples, + tuple, listClient, exceptionItems, services, @@ -36,7 +36,7 @@ export const threatMatchExecutor = async ({ wrapHits, }: { rule: SavedObject>; - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; listClient: ListClient; exceptionItems: ExceptionListItemSchema[]; services: AlertServices; @@ -51,7 +51,7 @@ export const threatMatchExecutor = async ({ const ruleParams = rule.attributes.params; const inputIndex = await getInputIndex(services, version, ruleParams.index); return createThreatSignals({ - tuples, + tuple, threatMapping: ruleParams.threatMapping, query: ruleParams.query, inputIndex, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index bb2e57b0606e59..a83af237f72963 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -23,7 +23,7 @@ import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } fr // search_after through documents and re-index using bulk endpoint. export const searchAfterAndBulkCreate = async ({ - tuples: totalToFromTuples, + tuple, ruleSO, exceptionsList, services, @@ -49,150 +49,143 @@ export const searchAfterAndBulkCreate = async ({ // to ensure we don't exceed maxSignals let signalsCreatedCount = 0; - const tuplesToBeLogged = [...totalToFromTuples]; - logger.debug(buildRuleMessage(`totalToFromTuples: ${totalToFromTuples.length}`)); - - while (totalToFromTuples.length > 0) { - const tuple = totalToFromTuples.pop(); - if (tuple == null || tuple.to == null || tuple.from == null) { - logger.error(buildRuleMessage(`[-] malformed date tuple`)); - return createSearchAfterReturnType({ - success: false, - errors: ['malformed date tuple'], - }); - } - signalsCreatedCount = 0; - while (signalsCreatedCount < tuple.maxSignals) { - try { - let mergedSearchResults = createSearchResultReturnType(); - logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); + if (tuple == null || tuple.to == null || tuple.from == null) { + logger.error(buildRuleMessage(`[-] malformed date tuple`)); + return createSearchAfterReturnType({ + success: false, + errors: ['malformed date tuple'], + }); + } + signalsCreatedCount = 0; + while (signalsCreatedCount < tuple.maxSignals) { + try { + let mergedSearchResults = createSearchResultReturnType(); + logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); - if (hasSortId) { - const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ - buildRuleMessage, - searchAfterSortIds: sortIds, - index: inputIndexPattern, - from: tuple.from.toISOString(), - to: tuple.to.toISOString(), - services, - logger, - // @ts-expect-error please, declare a type explicitly instead of unknown - filter, - pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), + if (hasSortId) { + const { searchResult, searchDuration, searchErrors } = await singleSearchAfter({ + buildRuleMessage, + searchAfterSortIds: sortIds, + index: inputIndexPattern, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), + services, + logger, + // @ts-expect-error please, declare a type explicitly instead of unknown + filter, + pageSize: Math.ceil(Math.min(tuple.maxSignals, pageSize)), + timestampOverride: ruleParams.timestampOverride, + }); + mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); + toReturn = mergeReturns([ + toReturn, + createSearchAfterReturnTypeFromResponse({ + searchResult: mergedSearchResults, timestampOverride: ruleParams.timestampOverride, - }); - mergedSearchResults = mergeSearchResults([mergedSearchResults, searchResult]); - toReturn = mergeReturns([ - toReturn, - createSearchAfterReturnTypeFromResponse({ - searchResult: mergedSearchResults, - timestampOverride: ruleParams.timestampOverride, - }), - createSearchAfterReturnType({ - searchAfterTimes: [searchDuration], - errors: searchErrors, - }), - ]); + }), + createSearchAfterReturnType({ + searchAfterTimes: [searchDuration], + errors: searchErrors, + }), + ]); - const lastSortIds = getSafeSortIds( - searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort - ); - if (lastSortIds != null && lastSortIds.length !== 0) { - sortIds = lastSortIds; - hasSortId = true; - } else { - hasSortId = false; - } + const lastSortIds = getSafeSortIds( + searchResult.hits.hits[searchResult.hits.hits.length - 1]?.sort + ); + if (lastSortIds != null && lastSortIds.length !== 0) { + sortIds = lastSortIds; + hasSortId = true; + } else { + hasSortId = false; } + } + + // determine if there are any candidate signals to be processed + const totalHits = createTotalHitsFromSearchResult({ searchResult: mergedSearchResults }); + logger.debug(buildRuleMessage(`totalHits: ${totalHits}`)); + logger.debug( + buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`) + ); - // determine if there are any candidate signals to be processed - const totalHits = createTotalHitsFromSearchResult({ searchResult: mergedSearchResults }); - logger.debug(buildRuleMessage(`totalHits: ${totalHits}`)); + if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { logger.debug( - buildRuleMessage(`searchResult.hit.hits.length: ${mergedSearchResults.hits.hits.length}`) + buildRuleMessage( + `${ + totalHits === 0 ? 'totalHits' : 'searchResult.hits.hits.length' + } was 0, exiting and moving on to next tuple` + ) ); + break; + } - if (totalHits === 0 || mergedSearchResults.hits.hits.length === 0) { - logger.debug( - buildRuleMessage( - `${ - totalHits === 0 ? 'totalHits' : 'searchResult.hits.hits.length' - } was 0, exiting and moving on to next tuple` - ) - ); - break; - } - - // filter out the search results that match with the values found in the list. - // the resulting set are signals to be indexed, given they are not duplicates - // of signals already present in the signals index. - const filteredEvents = await filterEventsAgainstList({ - listClient, - exceptionsList, - logger, - eventSearchResult: mergedSearchResults, - buildRuleMessage, - }); - - // only bulk create if there are filteredEvents leftover - // if there isn't anything after going through the value list filter - // skip the call to bulk create and proceed to the next search_after, - // if there is a sort id to continue the search_after with. - if (filteredEvents.hits.hits.length !== 0) { - // make sure we are not going to create more signals than maxSignals allows - if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { - filteredEvents.hits.hits = filteredEvents.hits.hits.slice( - 0, - tuple.maxSignals - signalsCreatedCount - ); - } - const enrichedEvents = await enrichment(filteredEvents); - const wrappedDocs = wrapHits(enrichedEvents.hits.hits); + // filter out the search results that match with the values found in the list. + // the resulting set are signals to be indexed, given they are not duplicates + // of signals already present in the signals index. + const filteredEvents = await filterEventsAgainstList({ + listClient, + exceptionsList, + logger, + eventSearchResult: mergedSearchResults, + buildRuleMessage, + }); - const { - bulkCreateDuration: bulkDuration, - createdItemsCount: createdCount, - createdItems, - success: bulkSuccess, - errors: bulkErrors, - } = await bulkCreate(wrappedDocs); - toReturn = mergeReturns([ - toReturn, - createSearchAfterReturnType({ - success: bulkSuccess, - createdSignalsCount: createdCount, - createdSignals: createdItems, - bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, - errors: bulkErrors, - }), - ]); - signalsCreatedCount += createdCount; - logger.debug(buildRuleMessage(`created ${createdCount} signals`)); - logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); - logger.debug( - buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) + // only bulk create if there are filteredEvents leftover + // if there isn't anything after going through the value list filter + // skip the call to bulk create and proceed to the next search_after, + // if there is a sort id to continue the search_after with. + if (filteredEvents.hits.hits.length !== 0) { + // make sure we are not going to create more signals than maxSignals allows + if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { + filteredEvents.hits.hits = filteredEvents.hits.hits.slice( + 0, + tuple.maxSignals - signalsCreatedCount ); - - sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); } + const enrichedEvents = await enrichment(filteredEvents); + const wrappedDocs = wrapHits(enrichedEvents.hits.hits); - if (!hasSortId) { - logger.debug(buildRuleMessage('ran out of sort ids to sort on')); - break; - } - } catch (exc: unknown) { - logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); - return mergeReturns([ + const { + bulkCreateDuration: bulkDuration, + createdItemsCount: createdCount, + createdItems, + success: bulkSuccess, + errors: bulkErrors, + } = await bulkCreate(wrappedDocs); + toReturn = mergeReturns([ toReturn, createSearchAfterReturnType({ - success: false, - errors: [`${exc}`], + success: bulkSuccess, + createdSignalsCount: createdCount, + createdSignals: createdItems, + bulkCreateTimes: bulkDuration ? [bulkDuration] : undefined, + errors: bulkErrors, }), ]); + signalsCreatedCount += createdCount; + logger.debug(buildRuleMessage(`created ${createdCount} signals`)); + logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); + logger.debug( + buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) + ); + + sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); + } + + if (!hasSortId) { + logger.debug(buildRuleMessage('ran out of sort ids to sort on')); + break; } + } catch (exc: unknown) { + logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); + return mergeReturns([ + toReturn, + createSearchAfterReturnType({ + success: false, + errors: [`${exc}`], + }), + ]); } } logger.debug(buildRuleMessage(`[+] completed bulk index of ${toReturn.createdSignalsCount}`)); - toReturn.totalToFromTuples = tuplesToBeLogged; return toReturn; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 288406ff32481a..5e58e85078a626 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -264,20 +264,22 @@ export const signalRulesAlertType = ({ } } else if (isThreatMatchRule(type)) { const threatRuleSO = asTypeSpecificSO(savedObject, threatRuleParams); - result = await threatMatchExecutor({ - rule: threatRuleSO, - tuples, - listClient, - exceptionItems, - services, - version, - searchAfterSize, - logger, - eventsTelemetry, - buildRuleMessage, - bulkCreate, - wrapHits, - }); + for (const tuple of tuples) { + result = await threatMatchExecutor({ + rule: threatRuleSO, + tuple, + listClient, + exceptionItems, + services, + version, + searchAfterSize, + logger, + eventsTelemetry, + buildRuleMessage, + bulkCreate, + wrapHits, + }); + } } else if (isQueryRule(type)) { const queryRuleSO = validateQueryRuleTypes(savedObject); result = await queryExecutor({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 3e30a08f1ae69c..806f5e47608e40 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -13,7 +13,7 @@ import { CreateThreatSignalOptions } from './types'; import { SearchAfterAndBulkCreateReturnType } from '../types'; export const createThreatSignal = async ({ - tuples, + tuple, threatMapping, threatEnrichment, query, @@ -70,7 +70,7 @@ export const createThreatSignal = async ({ ); const result = await searchAfterAndBulkCreate({ - tuples, + tuple, listClient, exceptionsList: exceptionItems, ruleSO, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 5054ab1b2cca50..169a820392a6e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -15,7 +15,7 @@ import { combineConcurrentResults } from './utils'; import { buildThreatEnrichment } from './build_threat_enrichment'; export const createThreatSignals = async ({ - tuples, + tuple, threatMapping, query, inputIndex, @@ -104,7 +104,7 @@ export const createThreatSignals = async ({ const concurrentSearchesPerformed = chunks.map>( (slicedChunk) => createThreatSignal({ - tuples, + tuple, threatEnrichment, threatMapping, query, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 34b064b0f88053..ded79fc647ac41 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -40,7 +40,7 @@ import { ThreatRuleParams } from '../../schemas/rule_schemas'; export type SortOrderOrUndefined = 'asc' | 'desc' | undefined; export interface CreateThreatSignalsOptions { - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; threatMapping: ThreatMapping; query: string; inputIndex: string[]; @@ -70,7 +70,7 @@ export interface CreateThreatSignalsOptions { } export interface CreateThreatSignalOptions { - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; threatMapping: ThreatMapping; threatEnrichment: SignalsEnrichment; query: string; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index c35eb04ba12707..8a6ce91b2575ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -262,11 +262,11 @@ export type WrapHits = ( ) => Array>; export interface SearchAfterAndBulkCreateParams { - tuples: Array<{ + tuple: { to: moment.Moment; from: moment.Moment; maxSignals: number; - }>; + }; ruleSO: SavedObject; services: AlertServices; listClient: ListClient; From 1c0b81d71a5d636b9e73fa1f2a8b43c31d88976a Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 8 Jun 2021 21:59:48 -0400 Subject: [PATCH 03/14] Modify custom query rules to receive a single date range tuple --- .../signals/executors/query.ts | 6 ++-- .../signals/signal_rule_alert_type.ts | 30 ++++++++++--------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts index 9d76a06afa2755..385c01c2f1cda1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts @@ -24,7 +24,7 @@ import { QueryRuleParams, SavedQueryRuleParams } from '../../schemas/rule_schema export const queryExecutor = async ({ rule, - tuples, + tuple, listClient, exceptionItems, services, @@ -37,7 +37,7 @@ export const queryExecutor = async ({ wrapHits, }: { rule: SavedObject>; - tuples: RuleRangeTuple[]; + tuple: RuleRangeTuple; listClient: ListClient; exceptionItems: ExceptionListItemSchema[]; services: AlertServices; @@ -63,7 +63,7 @@ export const queryExecutor = async ({ }); return searchAfterAndBulkCreate({ - tuples, + tuple, listClient, exceptionsList: exceptionItems, ruleSO: rule, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 5e58e85078a626..8c0f375182a3ed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -282,20 +282,22 @@ export const signalRulesAlertType = ({ } } else if (isQueryRule(type)) { const queryRuleSO = validateQueryRuleTypes(savedObject); - result = await queryExecutor({ - rule: queryRuleSO, - tuples, - listClient, - exceptionItems, - services, - version, - searchAfterSize, - logger, - eventsTelemetry, - buildRuleMessage, - bulkCreate, - wrapHits, - }); + for (const tuple of tuples) { + result = await queryExecutor({ + rule: queryRuleSO, + tuple, + listClient, + exceptionItems, + services, + version, + searchAfterSize, + logger, + eventsTelemetry, + buildRuleMessage, + bulkCreate, + wrapHits, + }); + } } else if (isEqlRule(type)) { const eqlRuleSO = asTypeSpecificSO(savedObject, eqlRuleParams); result = await eqlExecutor({ From 0578cc5e7f60571e3bd3f174bcb65484ad7e8fbb Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 8 Jun 2021 22:32:12 -0400 Subject: [PATCH 04/14] Fix up tests (partially) --- .../signals/executors/threshold.test.ts | 3 +- .../signals/search_after_bulk_create.test.ts | 32 +++++++++---------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index f03e8b8a147aea..6bb4e45e4c163b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import moment from 'moment'; import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { thresholdExecutor } from './threshold'; @@ -55,7 +56,7 @@ describe('threshold_executor', () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; const response = await thresholdExecutor({ rule: thresholdSO, - tuples: [], + tuple: { to: moment(), from: moment(), maxSignals: 100 }, exceptionItems, services: alertServices, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index e4eb7e854f670f..ee4f457ef3eb88 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -44,14 +44,14 @@ describe('searchAfterAndBulkCreate', () => { const sampleParams = getQueryRuleParams(); const ruleSO = sampleRuleSO(getQueryRuleParams()); sampleParams.maxSignals = 30; - let tuples: RuleRangeTuple[]; + let tuple: RuleRangeTuple; beforeEach(() => { jest.clearAllMocks(); listClient = listMock.getListClient(); listClient.searchListItemByValues = jest.fn().mockResolvedValue([]); inputIndexPattern = ['auditbeat-*']; mockService = alertsMock.createAlertServices(); - ({ tuples } = getRuleRangeTuples({ + tuple = getRuleRangeTuples({ logger: mockLogger, previousStartedAt: new Date(), from: sampleParams.from, @@ -59,7 +59,7 @@ describe('searchAfterAndBulkCreate', () => { interval: '5m', maxSignals: sampleParams.maxSignals, buildRuleMessage, - })); + }).tuples[0]; bulkCreate = bulkCreateFactory( mockLogger, mockService.scopedClusterClient.asCurrentUser, @@ -174,7 +174,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ - tuples, + tuple, ruleSO, listClient, exceptionsList: [exceptionItem], @@ -279,7 +279,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -357,7 +357,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -416,7 +416,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -495,7 +495,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [], services: mockService, @@ -550,7 +550,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -627,7 +627,7 @@ describe('searchAfterAndBulkCreate', () => { ]; const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [exceptionItem], services: mockService, @@ -701,7 +701,7 @@ describe('searchAfterAndBulkCreate', () => { ); const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [], services: mockService, @@ -746,7 +746,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ listClient, exceptionsList: [exceptionItem], - tuples, + tuple, ruleSO, services: mockService, logger: mockLogger, @@ -793,7 +793,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ listClient, exceptionsList: [exceptionItem], - tuples, + tuple, ruleSO, services: mockService, logger: mockLogger, @@ -854,7 +854,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ listClient, exceptionsList: [exceptionItem], - tuples, + tuple, ruleSO, services: mockService, logger: mockLogger, @@ -979,7 +979,7 @@ describe('searchAfterAndBulkCreate', () => { errors, } = await searchAfterAndBulkCreate({ ruleSO, - tuples, + tuple, listClient, exceptionsList: [], services: mockService, @@ -1075,7 +1075,7 @@ describe('searchAfterAndBulkCreate', () => { const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ enrichment: mockEnrichment, ruleSO, - tuples, + tuple, listClient, exceptionsList: [], services: mockService, From d1473306ec07d3466f4fbe0e890fe2f2121cd707 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Wed, 9 Jun 2021 11:44:39 -0400 Subject: [PATCH 05/14] Change log message to indicate single tuple instead of array --- .../lib/detection_engine/signals/search_after_bulk_create.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index a83af237f72963..eb4af0c38ce254 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -112,7 +112,7 @@ export const searchAfterAndBulkCreate = async ({ buildRuleMessage( `${ totalHits === 0 ? 'totalHits' : 'searchResult.hits.hits.length' - } was 0, exiting and moving on to next tuple` + } was 0, exiting early` ) ); break; From 1bd5eea665b6a9b8ddf77ab6a9d9cbc4b8a69c99 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Sat, 12 Jun 2021 22:19:39 -0400 Subject: [PATCH 06/14] Bad test? --- .../signals/search_after_bulk_create.test.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index ee4f457ef3eb88..184b49c2d6c7b9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -569,11 +569,6 @@ describe('searchAfterAndBulkCreate', () => { expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(createdSignalsCount).toEqual(0); // should not create any signals because all events were in the allowlist expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - // I don't like testing log statements since logs change but this is the best - // way I can think of to ensure this section is getting hit with this test case. - expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[7][0]).toContain( - 'ran out of sort ids to sort on name: "fake name" id: "fake id" rule id: "fake rule id" signals index: "fakeindex"' - ); }); test('should return success when no sortId present but search results are in the allowlist', async () => { From b30179356fb7eef78d0c14f627fff6adc1c6a9f9 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 14 Jun 2021 12:19:54 -0400 Subject: [PATCH 07/14] Prevent max_signals from being exceeded on threat match rule executions --- .../signals/search_after_bulk_create.ts | 41 +++++++++++++++++-- .../threat_mapping/create_threat_signal.ts | 2 + .../threat_mapping/create_threat_signals.ts | 5 +++ .../signals/threat_mapping/types.ts | 2 + .../lib/detection_engine/signals/types.ts | 5 +++ .../lib/detection_engine/signals/utils.ts | 20 +++++++++ 6 files changed, 71 insertions(+), 4 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index eb4af0c38ce254..be41e366c6d533 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -18,10 +18,13 @@ import { mergeReturns, mergeSearchResults, getSafeSortIds, + getLock, + releaseLock, } from './utils'; import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types'; // search_after through documents and re-index using bulk endpoint. +// eslint-disable-next-line complexity export const searchAfterAndBulkCreate = async ({ tuple, ruleSO, @@ -37,6 +40,7 @@ export const searchAfterAndBulkCreate = async ({ enrichment = identity, bulkCreate, wrapHits, + state, }: SearchAfterAndBulkCreateParams): Promise => { const ruleParams = ruleSO.attributes.params; let toReturn = createSearchAfterReturnType(); @@ -48,16 +52,22 @@ export const searchAfterAndBulkCreate = async ({ // signalsCreatedCount keeps track of how many signals we have created, // to ensure we don't exceed maxSignals let signalsCreatedCount = 0; + const signalsAlreadyCreated = () => state?.signalsCreated || 0; + const totalSignalsCreated = (_signalsCreatedCount: number): number => { + return _signalsCreatedCount + signalsAlreadyCreated(); + }; if (tuple == null || tuple.to == null || tuple.from == null) { logger.error(buildRuleMessage(`[-] malformed date tuple`)); + if (state != null) { + releaseLock(state); + } return createSearchAfterReturnType({ success: false, errors: ['malformed date tuple'], }); } - signalsCreatedCount = 0; - while (signalsCreatedCount < tuple.maxSignals) { + while (totalSignalsCreated(signalsCreatedCount) < tuple.maxSignals) { try { let mergedSearchResults = createSearchResultReturnType(); logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); @@ -134,11 +144,25 @@ export const searchAfterAndBulkCreate = async ({ // skip the call to bulk create and proceed to the next search_after, // if there is a sort id to continue the search_after with. if (filteredEvents.hits.hits.length !== 0) { + if (state != null) { + const error = await getLock(state); + if (error != null) { + logger.error(buildRuleMessage(error)); + return createSearchAfterReturnType({ + success: false, + errors: [error], + }); + } + } + // make sure we are not going to create more signals than maxSignals allows - if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { + if ( + totalSignalsCreated(signalsCreatedCount) + filteredEvents.hits.hits.length > + tuple.maxSignals + ) { filteredEvents.hits.hits = filteredEvents.hits.hits.slice( 0, - tuple.maxSignals - signalsCreatedCount + tuple.maxSignals - totalSignalsCreated(signalsCreatedCount) ); } const enrichedEvents = await enrichment(filteredEvents); @@ -162,6 +186,12 @@ export const searchAfterAndBulkCreate = async ({ }), ]); signalsCreatedCount += createdCount; + if (state != null) { + // Protected by lock + // eslint-disable-next-line require-atomic-updates + state.signalsCreated += createdCount; + releaseLock(state); + } logger.debug(buildRuleMessage(`created ${createdCount} signals`)); logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); logger.debug( @@ -177,6 +207,9 @@ export const searchAfterAndBulkCreate = async ({ } } catch (exc: unknown) { logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); + if (state != null) { + releaseLock(state); + } return mergeReturns([ toReturn, createSearchAfterReturnType({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 806f5e47608e40..3901db648b58ad 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -36,6 +36,7 @@ export const createThreatSignal = async ({ currentResult, bulkCreate, wrapHits, + signalState, }: CreateThreatSignalOptions): Promise => { const threatFilter = buildThreatMappingFilter({ threatMapping, @@ -86,6 +87,7 @@ export const createThreatSignal = async ({ enrichment: threatEnrichment, bulkCreate, wrapHits, + state: signalState, }); logger.debug( buildRuleMessage( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index 169a820392a6e6..b2b4c1c899f852 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -101,6 +101,10 @@ export const createThreatSignals = async ({ while (threatList.hits.hits.length !== 0) { const chunks = chunk(itemsPerSearch, threatList.hits.hits); logger.debug(buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`)); + const signalState = { + isLocked: false, + signalsCreated: 0, + }; const concurrentSearchesPerformed = chunks.map>( (slicedChunk) => createThreatSignal({ @@ -127,6 +131,7 @@ export const createThreatSignals = async ({ currentResult: results, bulkCreate, wrapHits, + signalState, }) ); const searchesPerformed = await Promise.all(concurrentSearchesPerformed); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index ded79fc647ac41..0c4a4869fbc446 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -32,6 +32,7 @@ import { BulkCreate, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, + SearchAfterAndBulkCreateState, SignalsEnrichment, WrapHits, } from '../types'; @@ -93,6 +94,7 @@ export interface CreateThreatSignalOptions { currentResult: SearchAfterAndBulkCreateReturnType; bulkCreate: BulkCreate; wrapHits: WrapHits; + signalState: SearchAfterAndBulkCreateState; } export interface BuildThreatMappingFilterOptions { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 8a6ce91b2575ab..d29f99a48c12df 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -261,6 +261,10 @@ export type WrapHits = ( hits: Array> ) => Array>; +export interface SearchAfterAndBulkCreateState { + isLocked: boolean; + signalsCreated: number; +} export interface SearchAfterAndBulkCreateParams { tuple: { to: moment.Moment; @@ -282,6 +286,7 @@ export interface SearchAfterAndBulkCreateParams { enrichment?: SignalsEnrichment; bulkCreate: BulkCreate; wrapHits: WrapHits; + state?: SearchAfterAndBulkCreateState | undefined; } export interface SearchAfterAndBulkCreateReturnType { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 6d67bab6eb2f76..1220e637a76da8 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -914,3 +914,23 @@ export const buildChunkedOrFilter = (field: string, values: string[], chunkSize: }) .join(' OR '); }; + +const LOCK_INTERVAL_MS = 250; +const LOCK_MAX_TRIES = 100; +const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +export const getLock = async (signalState: { isLocked: boolean }): Promise => { + let tries = 0; + while (signalState.isLocked && tries < LOCK_MAX_TRIES) { + await sleep(LOCK_INTERVAL_MS); + tries++; + } + if (!signalState.isLocked) { + signalState.isLocked = true; + } + return `Error retrieving lock after {tries} tries.`; +}; + +export const releaseLock = (signalState: { isLocked: boolean }) => { + signalState.isLocked = false; +}; From 4752688ef99c9947c21621018d990c61cf07a830 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 14 Jun 2021 16:51:59 -0400 Subject: [PATCH 08/14] Revert "Prevent max_signals from being exceeded on threat match rule executions" This reverts commit ba3b2f7a382ef7c369f02c7939e1495f72d92bfe. --- .../signals/search_after_bulk_create.ts | 41 ++----------------- .../threat_mapping/create_threat_signal.ts | 2 - .../threat_mapping/create_threat_signals.ts | 5 --- .../signals/threat_mapping/types.ts | 2 - .../lib/detection_engine/signals/types.ts | 5 --- .../lib/detection_engine/signals/utils.ts | 20 --------- 6 files changed, 4 insertions(+), 71 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index be41e366c6d533..eb4af0c38ce254 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -18,13 +18,10 @@ import { mergeReturns, mergeSearchResults, getSafeSortIds, - getLock, - releaseLock, } from './utils'; import { SearchAfterAndBulkCreateParams, SearchAfterAndBulkCreateReturnType } from './types'; // search_after through documents and re-index using bulk endpoint. -// eslint-disable-next-line complexity export const searchAfterAndBulkCreate = async ({ tuple, ruleSO, @@ -40,7 +37,6 @@ export const searchAfterAndBulkCreate = async ({ enrichment = identity, bulkCreate, wrapHits, - state, }: SearchAfterAndBulkCreateParams): Promise => { const ruleParams = ruleSO.attributes.params; let toReturn = createSearchAfterReturnType(); @@ -52,22 +48,16 @@ export const searchAfterAndBulkCreate = async ({ // signalsCreatedCount keeps track of how many signals we have created, // to ensure we don't exceed maxSignals let signalsCreatedCount = 0; - const signalsAlreadyCreated = () => state?.signalsCreated || 0; - const totalSignalsCreated = (_signalsCreatedCount: number): number => { - return _signalsCreatedCount + signalsAlreadyCreated(); - }; if (tuple == null || tuple.to == null || tuple.from == null) { logger.error(buildRuleMessage(`[-] malformed date tuple`)); - if (state != null) { - releaseLock(state); - } return createSearchAfterReturnType({ success: false, errors: ['malformed date tuple'], }); } - while (totalSignalsCreated(signalsCreatedCount) < tuple.maxSignals) { + signalsCreatedCount = 0; + while (signalsCreatedCount < tuple.maxSignals) { try { let mergedSearchResults = createSearchResultReturnType(); logger.debug(buildRuleMessage(`sortIds: ${sortIds}`)); @@ -144,25 +134,11 @@ export const searchAfterAndBulkCreate = async ({ // skip the call to bulk create and proceed to the next search_after, // if there is a sort id to continue the search_after with. if (filteredEvents.hits.hits.length !== 0) { - if (state != null) { - const error = await getLock(state); - if (error != null) { - logger.error(buildRuleMessage(error)); - return createSearchAfterReturnType({ - success: false, - errors: [error], - }); - } - } - // make sure we are not going to create more signals than maxSignals allows - if ( - totalSignalsCreated(signalsCreatedCount) + filteredEvents.hits.hits.length > - tuple.maxSignals - ) { + if (signalsCreatedCount + filteredEvents.hits.hits.length > tuple.maxSignals) { filteredEvents.hits.hits = filteredEvents.hits.hits.slice( 0, - tuple.maxSignals - totalSignalsCreated(signalsCreatedCount) + tuple.maxSignals - signalsCreatedCount ); } const enrichedEvents = await enrichment(filteredEvents); @@ -186,12 +162,6 @@ export const searchAfterAndBulkCreate = async ({ }), ]); signalsCreatedCount += createdCount; - if (state != null) { - // Protected by lock - // eslint-disable-next-line require-atomic-updates - state.signalsCreated += createdCount; - releaseLock(state); - } logger.debug(buildRuleMessage(`created ${createdCount} signals`)); logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); logger.debug( @@ -207,9 +177,6 @@ export const searchAfterAndBulkCreate = async ({ } } catch (exc: unknown) { logger.error(buildRuleMessage(`[-] search_after and bulk threw an error ${exc}`)); - if (state != null) { - releaseLock(state); - } return mergeReturns([ toReturn, createSearchAfterReturnType({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 3901db648b58ad..806f5e47608e40 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -36,7 +36,6 @@ export const createThreatSignal = async ({ currentResult, bulkCreate, wrapHits, - signalState, }: CreateThreatSignalOptions): Promise => { const threatFilter = buildThreatMappingFilter({ threatMapping, @@ -87,7 +86,6 @@ export const createThreatSignal = async ({ enrichment: threatEnrichment, bulkCreate, wrapHits, - state: signalState, }); logger.debug( buildRuleMessage( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index b2b4c1c899f852..169a820392a6e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -101,10 +101,6 @@ export const createThreatSignals = async ({ while (threatList.hits.hits.length !== 0) { const chunks = chunk(itemsPerSearch, threatList.hits.hits); logger.debug(buildRuleMessage(`${chunks.length} concurrent indicator searches are starting.`)); - const signalState = { - isLocked: false, - signalsCreated: 0, - }; const concurrentSearchesPerformed = chunks.map>( (slicedChunk) => createThreatSignal({ @@ -131,7 +127,6 @@ export const createThreatSignals = async ({ currentResult: results, bulkCreate, wrapHits, - signalState, }) ); const searchesPerformed = await Promise.all(concurrentSearchesPerformed); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index 0c4a4869fbc446..ded79fc647ac41 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -32,7 +32,6 @@ import { BulkCreate, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, - SearchAfterAndBulkCreateState, SignalsEnrichment, WrapHits, } from '../types'; @@ -94,7 +93,6 @@ export interface CreateThreatSignalOptions { currentResult: SearchAfterAndBulkCreateReturnType; bulkCreate: BulkCreate; wrapHits: WrapHits; - signalState: SearchAfterAndBulkCreateState; } export interface BuildThreatMappingFilterOptions { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index d29f99a48c12df..8a6ce91b2575ab 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -261,10 +261,6 @@ export type WrapHits = ( hits: Array> ) => Array>; -export interface SearchAfterAndBulkCreateState { - isLocked: boolean; - signalsCreated: number; -} export interface SearchAfterAndBulkCreateParams { tuple: { to: moment.Moment; @@ -286,7 +282,6 @@ export interface SearchAfterAndBulkCreateParams { enrichment?: SignalsEnrichment; bulkCreate: BulkCreate; wrapHits: WrapHits; - state?: SearchAfterAndBulkCreateState | undefined; } export interface SearchAfterAndBulkCreateReturnType { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 1220e637a76da8..6d67bab6eb2f76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -914,23 +914,3 @@ export const buildChunkedOrFilter = (field: string, values: string[], chunkSize: }) .join(' OR '); }; - -const LOCK_INTERVAL_MS = 250; -const LOCK_MAX_TRIES = 100; -const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - -export const getLock = async (signalState: { isLocked: boolean }): Promise => { - let tries = 0; - while (signalState.isLocked && tries < LOCK_MAX_TRIES) { - await sleep(LOCK_INTERVAL_MS); - tries++; - } - if (!signalState.isLocked) { - signalState.isLocked = true; - } - return `Error retrieving lock after {tries} tries.`; -}; - -export const releaseLock = (signalState: { isLocked: boolean }) => { - signalState.isLocked = false; -}; From 5a0739ac06000b5559b28bff24b416763613549b Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 14 Jun 2021 16:56:42 -0400 Subject: [PATCH 09/14] Modify EQL rules to use date range tuple --- .../detection_engine/signals/executors/eql.ts | 7 +++++-- .../signals/signal_rule_alert_type.ts | 21 +++++++++++-------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts index 28d1f3e19baeed..a187b730696829 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -28,6 +28,7 @@ import { AlertAttributes, BulkCreate, EqlSignalSearchResponse, + RuleRangeTuple, SearchAfterAndBulkCreateReturnType, WrappedSignalHit, } from '../types'; @@ -35,6 +36,7 @@ import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../uti export const eqlExecutor = async ({ rule, + tuple, exceptionItems, services, version, @@ -43,6 +45,7 @@ export const eqlExecutor = async ({ bulkCreate, }: { rule: SavedObject>; + tuple: RuleRangeTuple; exceptionItems: ExceptionListItemSchema[]; services: AlertServices; version: string; @@ -81,8 +84,8 @@ export const eqlExecutor = async ({ const request = buildEqlSearchRequest( ruleParams.query, inputIndex, - ruleParams.from, - ruleParams.to, + tuple.from.toISOString(), + tuple.to.toISOString(), searchAfterSize, ruleParams.timestampOverride, exceptionItems, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 8c0f375182a3ed..493ea1d8585b15 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -300,15 +300,18 @@ export const signalRulesAlertType = ({ } } else if (isEqlRule(type)) { const eqlRuleSO = asTypeSpecificSO(savedObject, eqlRuleParams); - result = await eqlExecutor({ - rule: eqlRuleSO, - exceptionItems, - services, - version, - searchAfterSize, - bulkCreate, - logger, - }); + for (const tuple of tuples) { + result = await eqlExecutor({ + rule: eqlRuleSO, + tuple, + exceptionItems, + services, + version, + searchAfterSize, + bulkCreate, + logger, + }); + } } else { throw new Error(`unknown rule type ${type}`); } From 1038103abb1f1d845fcce322cdf80efad3ff2364 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 14 Jun 2021 17:08:06 -0400 Subject: [PATCH 10/14] Modify ML rules to use date range tuple --- .../detection_engine/signals/executors/ml.ts | 8 +++--- .../signals/signal_rule_alert_type.ts | 25 +++++++++++-------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts index f5c7d8822b51f0..20c4cb16dadc8d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts @@ -21,11 +21,12 @@ import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; import { filterEventsAgainstList } from '../filters/filter_events_against_list'; import { findMlSignals } from '../find_ml_signals'; import { BuildRuleMessage } from '../rule_messages'; -import { AlertAttributes, BulkCreate, WrapHits } from '../types'; +import { AlertAttributes, BulkCreate, RuleRangeTuple, WrapHits } from '../types'; import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../utils'; export const mlExecutor = async ({ rule, + tuple, ml, listClient, exceptionItems, @@ -36,6 +37,7 @@ export const mlExecutor = async ({ wrapHits, }: { rule: SavedObject>; + tuple: RuleRangeTuple; ml: SetupPlugins['ml']; listClient: ListClient; exceptionItems: ExceptionListItemSchema[]; @@ -88,8 +90,8 @@ export const mlExecutor = async ({ savedObjectsClient: services.savedObjectsClient, jobIds: ruleParams.machineLearningJobId, anomalyThreshold: ruleParams.anomalyThreshold, - from: ruleParams.from, - to: ruleParams.to, + from: tuple.from.toISOString(), + to: tuple.to.toISOString(), exceptionItems, }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 493ea1d8585b15..bb1e50c14d4014 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -235,17 +235,20 @@ export const signalRulesAlertType = ({ if (isMlRule(type)) { const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams); - result = await mlExecutor({ - rule: mlRuleSO, - ml, - listClient, - exceptionItems, - services, - logger, - buildRuleMessage, - bulkCreate, - wrapHits, - }); + for (const tuple of tuples) { + result = await mlExecutor({ + rule: mlRuleSO, + tuple, + ml, + listClient, + exceptionItems, + services, + logger, + buildRuleMessage, + bulkCreate, + wrapHits, + }); + } } else if (isThresholdRule(type)) { const thresholdRuleSO = asTypeSpecificSO(savedObject, thresholdRuleParams); for (const tuple of tuples) { From 2f9ac90de3150aa2537f47f53f376e0e7eb889a0 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Mon, 14 Jun 2021 17:08:13 -0400 Subject: [PATCH 11/14] Fix ML/EQL tests --- .../lib/detection_engine/signals/executors/eql.test.ts | 6 +++++- .../lib/detection_engine/signals/executors/ml.test.ts | 8 +++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts index 947e7d573173ea..ab0a32ba5481cc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import moment from 'moment'; import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { eqlExecutor } from './eql'; @@ -23,6 +24,7 @@ describe('eql_executor', () => { let logger: ReturnType; let alertServices: AlertServicesMock; (getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION); + const params = getEqlRuleParams(); const eqlSO = { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', type: 'alert', @@ -40,10 +42,11 @@ describe('eql_executor', () => { interval: '5m', }, throttle: 'no_actions', - params: getEqlRuleParams(), + params, }, references: [], }; + const tuple = { from: moment(params.from), to: moment(params.to), maxSignals: params.maxSignals }; const searchAfterSize = 7; beforeEach(() => { @@ -64,6 +67,7 @@ describe('eql_executor', () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; const response = await eqlExecutor({ rule: eqlSO, + tuple, exceptionItems, services: alertServices, version, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts index 25a9d2c3f510fe..d83e3d91ff978d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts @@ -5,6 +5,7 @@ * 2.0. */ +import moment from 'moment'; import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { mlExecutor } from './ml'; @@ -26,7 +27,9 @@ describe('ml_executor', () => { const exceptionItems = [getExceptionListItemSchemaMock()]; let logger: ReturnType; let alertServices: AlertServicesMock; - const mlSO = sampleRuleSO(getMlRuleParams()); + const params = getMlRuleParams(); + const mlSO = sampleRuleSO(params); + const tuple = { from: moment(params.from), to: moment(params.to), maxSignals: params.maxSignals }; const buildRuleMessage = buildRuleMessageFactory({ id: mlSO.id, ruleId: mlSO.attributes.params.ruleId, @@ -60,6 +63,7 @@ describe('ml_executor', () => { await expect( mlExecutor({ rule: mlSO, + tuple, ml: undefined, exceptionItems, services: alertServices, @@ -76,6 +80,7 @@ describe('ml_executor', () => { jobsSummaryMock.mockResolvedValue([]); const response = await mlExecutor({ rule: mlSO, + tuple, ml: mlMock, exceptionItems, services: alertServices, @@ -101,6 +106,7 @@ describe('ml_executor', () => { const response = await mlExecutor({ rule: mlSO, + tuple, ml: mlMock, exceptionItems, services: alertServices, From 956963e74fcfe6a50071a3640a9045db0bb8c991 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 15 Jun 2021 12:39:25 -0400 Subject: [PATCH 12/14] Use dateMath to parse moments in ML/Threshold tests --- .../detection_engine/signals/executors/ml.test.ts | 8 ++++++-- .../signals/executors/threshold.test.ts | 12 +++++++++--- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts index d83e3d91ff978d..89c1392cb67ba7 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import moment from 'moment'; +import dateMath from '@elastic/datemath'; import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { mlExecutor } from './ml'; @@ -29,7 +29,11 @@ describe('ml_executor', () => { let alertServices: AlertServicesMock; const params = getMlRuleParams(); const mlSO = sampleRuleSO(params); - const tuple = { from: moment(params.from), to: moment(params.to), maxSignals: params.maxSignals }; + const tuple = { + from: dateMath.parse(params.from)!, + to: dateMath.parse(params.to)!, + maxSignals: params.maxSignals, + }; const buildRuleMessage = buildRuleMessageFactory({ id: mlSO.id, ruleId: mlSO.attributes.params.ruleId, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index 6bb4e45e4c163b..37ae45665001e6 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import moment from 'moment'; +import dateMath from '@elastic/datemath'; import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { thresholdExecutor } from './threshold'; @@ -18,6 +18,7 @@ describe('threshold_executor', () => { const version = '8.0.0'; let logger: ReturnType; let alertServices: AlertServicesMock; + const params = getThresholdRuleParams(); const thresholdSO = { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', type: 'alert', @@ -35,10 +36,15 @@ describe('threshold_executor', () => { interval: '5m', }, throttle: 'no_actions', - params: getThresholdRuleParams(), + params, }, references: [], }; + const tuple = { + from: dateMath.parse(params.from)!, + to: dateMath.parse(params.to)!, + maxSignals: params.maxSignals, + }; const buildRuleMessage = buildRuleMessageFactory({ id: thresholdSO.id, ruleId: thresholdSO.attributes.params.ruleId, @@ -56,7 +62,7 @@ describe('threshold_executor', () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; const response = await thresholdExecutor({ rule: thresholdSO, - tuple: { to: moment(), from: moment(), maxSignals: 100 }, + tuple, exceptionItems, services: alertServices, version, From feba6b5b57e456e3416538c141fcf170dc317141 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 15 Jun 2021 12:53:55 -0400 Subject: [PATCH 13/14] Add mocks for threshold test --- .../signals/executors/threshold.test.ts | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index 37ae45665001e6..3906c669222386 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -7,12 +7,15 @@ import dateMath from '@elastic/datemath'; import { loggingSystemMock } from 'src/core/server/mocks'; +// eslint-disable-next-line @kbn/eslint/no-restricted-paths +import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { thresholdExecutor } from './threshold'; import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock'; import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock'; import { buildRuleMessageFactory } from '../rule_messages'; +import { sampleEmptyDocSearchResults } from '../__mocks__/es_results'; describe('threshold_executor', () => { const version = '8.0.0'; @@ -54,6 +57,9 @@ describe('threshold_executor', () => { beforeEach(() => { alertServices = alertsMock.createAlertServices(); + alertServices.scopedClusterClient.asCurrentUser.search.mockResolvedValue( + elasticsearchClientMock.createSuccessTransportRequestPromise(sampleEmptyDocSearchResults()) + ); logger = loggingSystemMock.createLogger(); }); @@ -69,7 +75,13 @@ describe('threshold_executor', () => { logger, buildRuleMessage, startedAt: new Date(), - bulkCreate: jest.fn(), + bulkCreate: jest.fn().mockImplementation((hits) => ({ + errors: [], + success: true, + bulkCreateDuration: '0', + createdItemsCount: 0, + createdItems: [], + })), wrapHits: jest.fn(), }); expect(response.warningMessages.length).toEqual(1); From e3f8f8dd5c3f58d822f486f514551700b2b0e454 Mon Sep 17 00:00:00 2001 From: Madison Caldwell Date: Tue, 15 Jun 2021 12:55:14 -0400 Subject: [PATCH 14/14] Use dateMath for eql tests --- .../lib/detection_engine/signals/executors/eql.test.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts index ab0a32ba5481cc..e7af3d484dfbd2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts @@ -5,7 +5,7 @@ * 2.0. */ -import moment from 'moment'; +import dateMath from '@elastic/datemath'; import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; import { eqlExecutor } from './eql'; @@ -46,7 +46,11 @@ describe('eql_executor', () => { }, references: [], }; - const tuple = { from: moment(params.from), to: moment(params.to), maxSignals: params.maxSignals }; + const tuple = { + from: dateMath.parse(params.from)!, + to: dateMath.parse(params.to)!, + maxSignals: params.maxSignals, + }; const searchAfterSize = 7; beforeEach(() => {