[RAC][Security Solution] Pull Gap Remediation out of search_after_bul…

…k_create (elastic#102104) (elastic#102739)

* Modify threshold rules to receive a single date range tuple

* Modify threat match rules to receive a single date range tuple

* Modify custom query rules to receive a single date range tuple

* Fix up tests (partially)

* Change log message to indicate single tuple instead of array

* Bad test?

* Prevent max_signals from being exceeded on threat match rule executions

* Revert "Prevent max_signals from being exceeded on threat match rule executions"

This reverts commit ba3b2f7.

* Modify EQL rules to use date range tuple

* Modify ML rules to use date range tuple

* Fix ML/EQL tests

* Use dateMath to parse moments in ML/Threshold tests

* Add mocks for threshold test

* Use dateMath for eql tests

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import dateMath from '@elastic/datemath';
 import { loggingSystemMock } from 'src/core/server/mocks';
 import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks';
 import { eqlExecutor } from './eql';
@@ -23,6 +24,7 @@ describe('eql_executor', () => {
   let logger: ReturnType<typeof loggingSystemMock.createLogger>;
   let alertServices: AlertServicesMock;
   (getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION);
+  const params = getEqlRuleParams();
   const eqlSO = {
     id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
     type: 'alert',
@@ -40,10 +42,15 @@ describe('eql_executor', () => {
         interval: '5m',
       },
       throttle: 'no_actions',
-      params: getEqlRuleParams(),
+      params,
     },
     references: [],
   };
+  const tuple = {
+    from: dateMath.parse(params.from)!,
+    to: dateMath.parse(params.to)!,
+    maxSignals: params.maxSignals,
+  };
   const searchAfterSize = 7;
 
   beforeEach(() => {
@@ -64,6 +71,7 @@ describe('eql_executor', () => {
       const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })];
       const response = await eqlExecutor({
         rule: eqlSO,
+        tuple,
         exceptionItems,
         services: alertServices,
         version,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts

@@ -28,13 +28,15 @@ import {
   AlertAttributes,
   BulkCreate,
   EqlSignalSearchResponse,
+  RuleRangeTuple,
   SearchAfterAndBulkCreateReturnType,
   WrappedSignalHit,
 } from '../types';
 import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../utils';
 
 export const eqlExecutor = async ({
   rule,
+  tuple,
   exceptionItems,
   services,
   version,
@@ -43,6 +45,7 @@ export const eqlExecutor = async ({
   bulkCreate,
 }: {
   rule: SavedObject<AlertAttributes<EqlRuleParams>>;
+  tuple: RuleRangeTuple;
   exceptionItems: ExceptionListItemSchema[];
   services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
   version: string;
@@ -81,8 +84,8 @@ export const eqlExecutor = async ({
   const request = buildEqlSearchRequest(
     ruleParams.query,
     inputIndex,
-    ruleParams.from,
-    ruleParams.to,
+    tuple.from.toISOString(),
+    tuple.to.toISOString(),
     searchAfterSize,
     ruleParams.timestampOverride,
     exceptionItems,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts

@@ -5,6 +5,7 @@
  * 2.0.
  */
 
+import dateMath from '@elastic/datemath';
 import { loggingSystemMock } from 'src/core/server/mocks';
 import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks';
 import { mlExecutor } from './ml';
@@ -26,7 +27,13 @@ describe('ml_executor', () => {
   const exceptionItems = [getExceptionListItemSchemaMock()];
   let logger: ReturnType<typeof loggingSystemMock.createLogger>;
   let alertServices: AlertServicesMock;
-  const mlSO = sampleRuleSO(getMlRuleParams());
+  const params = getMlRuleParams();
+  const mlSO = sampleRuleSO(params);
+  const tuple = {
+    from: dateMath.parse(params.from)!,
+    to: dateMath.parse(params.to)!,
+    maxSignals: params.maxSignals,
+  };
   const buildRuleMessage = buildRuleMessageFactory({
     id: mlSO.id,
     ruleId: mlSO.attributes.params.ruleId,
@@ -60,6 +67,7 @@ describe('ml_executor', () => {
     await expect(
       mlExecutor({
         rule: mlSO,
+        tuple,
         ml: undefined,
         exceptionItems,
         services: alertServices,
@@ -76,6 +84,7 @@ describe('ml_executor', () => {
     jobsSummaryMock.mockResolvedValue([]);
     const response = await mlExecutor({
       rule: mlSO,
+      tuple,
       ml: mlMock,
       exceptionItems,
       services: alertServices,
@@ -101,6 +110,7 @@ describe('ml_executor', () => {
 
     const response = await mlExecutor({
       rule: mlSO,
+      tuple,
       ml: mlMock,
       exceptionItems,
       services: alertServices,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts

@@ -21,11 +21,12 @@ import { bulkCreateMlSignals } from '../bulk_create_ml_signals';
 import { filterEventsAgainstList } from '../filters/filter_events_against_list';
 import { findMlSignals } from '../find_ml_signals';
 import { BuildRuleMessage } from '../rule_messages';
-import { AlertAttributes, BulkCreate, WrapHits } from '../types';
+import { AlertAttributes, BulkCreate, RuleRangeTuple, WrapHits } from '../types';
 import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../utils';
 
 export const mlExecutor = async ({
   rule,
+  tuple,
   ml,
   listClient,
   exceptionItems,
@@ -36,6 +37,7 @@ export const mlExecutor = async ({
   wrapHits,
 }: {
   rule: SavedObject<AlertAttributes<MachineLearningRuleParams>>;
+  tuple: RuleRangeTuple;
   ml: SetupPlugins['ml'];
   listClient: ListClient;
   exceptionItems: ExceptionListItemSchema[];
@@ -88,8 +90,8 @@ export const mlExecutor = async ({
     savedObjectsClient: services.savedObjectsClient,
     jobIds: ruleParams.machineLearningJobId,
     anomalyThreshold: ruleParams.anomalyThreshold,
-    from: ruleParams.from,
-    to: ruleParams.to,
+    from: tuple.from.toISOString(),
+    to: tuple.to.toISOString(),
     exceptionItems,
   });
 

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts

@@ -24,7 +24,7 @@ import { QueryRuleParams, SavedQueryRuleParams } from '../../schemas/rule_schema
 
 export const queryExecutor = async ({
   rule,
-  tuples,
+  tuple,
   listClient,
   exceptionItems,
   services,
@@ -37,7 +37,7 @@ export const queryExecutor = async ({
   wrapHits,
 }: {
   rule: SavedObject<AlertAttributes<QueryRuleParams | SavedQueryRuleParams>>;
-  tuples: RuleRangeTuple[];
+  tuple: RuleRangeTuple;
   listClient: ListClient;
   exceptionItems: ExceptionListItemSchema[];
   services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
@@ -63,7 +63,7 @@ export const queryExecutor = async ({
   });
 
   return searchAfterAndBulkCreate({
-    tuples,
+    tuple,
     listClient,
     exceptionsList: exceptionItems,
     ruleSO: rule,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts

@@ -23,7 +23,7 @@ import { ThreatRuleParams } from '../../schemas/rule_schemas';
 
 export const threatMatchExecutor = async ({
   rule,
-  tuples,
+  tuple,
   listClient,
   exceptionItems,
   services,
@@ -36,7 +36,7 @@ export const threatMatchExecutor = async ({
   wrapHits,
 }: {
   rule: SavedObject<AlertAttributes<ThreatRuleParams>>;
-  tuples: RuleRangeTuple[];
+  tuple: RuleRangeTuple;
   listClient: ListClient;
   exceptionItems: ExceptionListItemSchema[];
   services: AlertServices<AlertInstanceState, AlertInstanceContext, 'default'>;
@@ -51,7 +51,7 @@ export const threatMatchExecutor = async ({
   const ruleParams = rule.attributes.params;
   const inputIndex = await getInputIndex(services, version, ruleParams.index);
   return createThreatSignals({
-    tuples,
+    tuple,
     threatMapping: ruleParams.threatMapping,
     query: ruleParams.query,
     inputIndex,

x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts

@@ -5,18 +5,23 @@
  * 2.0.
  */
 
+import dateMath from '@elastic/datemath';
 import { loggingSystemMock } from 'src/core/server/mocks';
+// eslint-disable-next-line @kbn/eslint/no-restricted-paths
+import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks';
 import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks';
 import { thresholdExecutor } from './threshold';
 import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock';
 import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock';
 import { getThresholdRuleParams } from '../../schemas/rule_schemas.mock';
 import { buildRuleMessageFactory } from '../rule_messages';
+import { sampleEmptyDocSearchResults } from '../__mocks__/es_results';
 
 describe('threshold_executor', () => {
   const version = '8.0.0';
   let logger: ReturnType<typeof loggingSystemMock.createLogger>;
   let alertServices: AlertServicesMock;
+  const params = getThresholdRuleParams();
   const thresholdSO = {
     id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
     type: 'alert',
@@ -34,10 +39,15 @@ describe('threshold_executor', () => {
         interval: '5m',
       },
       throttle: 'no_actions',
-      params: getThresholdRuleParams(),
+      params,
     },
     references: [],
   };
+  const tuple = {
+    from: dateMath.parse(params.from)!,
+    to: dateMath.parse(params.to)!,
+    maxSignals: params.maxSignals,
+  };
   const buildRuleMessage = buildRuleMessageFactory({
     id: thresholdSO.id,
     ruleId: thresholdSO.attributes.params.ruleId,
@@ -47,6 +57,9 @@ describe('threshold_executor', () => {
 
   beforeEach(() => {
     alertServices = alertsMock.createAlertServices();
+    alertServices.scopedClusterClient.asCurrentUser.search.mockResolvedValue(
+      elasticsearchClientMock.createSuccessTransportRequestPromise(sampleEmptyDocSearchResults())
+    );
     logger = loggingSystemMock.createLogger();
   });
 
@@ -55,14 +68,20 @@ describe('threshold_executor', () => {
       const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })];
       const response = await thresholdExecutor({
         rule: thresholdSO,
-        tuples: [],
+        tuple,
         exceptionItems,
         services: alertServices,
         version,
         logger,
         buildRuleMessage,
         startedAt: new Date(),
-        bulkCreate: jest.fn(),
+        bulkCreate: jest.fn().mockImplementation((hits) => ({
+          errors: [],
+          success: true,
+          bulkCreateDuration: '0',
+          createdItemsCount: 0,
+          createdItems: [],
+        })),
         wrapHits: jest.fn(),
       });
       expect(response.warningMessages.length).toEqual(1);