Updates so client findBulk query and removes alertId

x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts

@@ -479,7 +479,6 @@ export const getRuleExecutionStatuses = (): Array<
     type: 'my-type',
     id: 'e0b86950-4e9f-11ea-bdbd-07b56aa159b3',
     attributes: {
-      alertId: '04128c15-0d1b-4716-a4c5-46997ac7f3bc',
       statusDate: '2020-02-18T15:26:49.783Z',
       status: RuleExecutionStatus.succeeded,
       lastFailureAt: undefined,
@@ -492,15 +491,20 @@ export const getRuleExecutionStatuses = (): Array<
       bulkCreateTimeDurations: ['800.43'],
     },
     score: 1,
-    references: [],
+    references: [
+      {
+        id: '04128c15-0d1b-4716-a4c5-46997ac7f3bc',
+        type: 'alert',
+        name: 'alert_0',
+      },
+    ],
     updated_at: '2020-02-18T15:26:51.333Z',
     version: 'WzQ2LDFd',
   },
   {
     type: 'my-type',
     id: '91246bd0-5261-11ea-9650-33b954270f67',
     attributes: {
-      alertId: '1ea5a820-4da1-4e82-92a1-2b43a7bece08',
       statusDate: '2020-02-18T15:15:58.806Z',
       status: RuleExecutionStatus.failed,
       lastFailureAt: '2020-02-18T15:15:58.806Z',
@@ -514,7 +518,13 @@ export const getRuleExecutionStatuses = (): Array<
       bulkCreateTimeDurations: ['800.43'],
     },
     score: 1,
-    references: [],
+    references: [
+      {
+        id: '1ea5a820-4da1-4e82-92a1-2b43a7bece08',
+        type: 'alert',
+        name: 'alert_0',
+      },
+    ],
     updated_at: '2020-02-18T15:15:58.860Z',
     version: 'WzMyLDFd',
   },
@@ -523,7 +533,6 @@ export const getRuleExecutionStatuses = (): Array<
 export const getFindBulkResultStatus = (): FindBulkExecutionLogResponse => ({
   '04128c15-0d1b-4716-a4c5-46997ac7f3bd': [
     {
-      alertId: '04128c15-0d1b-4716-a4c5-46997ac7f3bd',
       statusDate: '2020-02-18T15:26:49.783Z',
       status: RuleExecutionStatus.succeeded,
       lastFailureAt: undefined,
@@ -538,7 +547,6 @@ export const getFindBulkResultStatus = (): FindBulkExecutionLogResponse => ({
   ],
   '1ea5a820-4da1-4e82-92a1-2b43a7bece08': [
     {
-      alertId: '1ea5a820-4da1-4e82-92a1-2b43a7bece08',
       statusDate: '2020-02-18T15:15:58.806Z',
       status: RuleExecutionStatus.failed,
       lastFailureAt: '2020-02-18T15:15:58.806Z',

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/rule_status_saved_objects_client.ts

@@ -14,6 +14,7 @@ import {
   SavedObjectsFindResult,
   SavedObjectsFindOptionsReference,
   SavedObjectsCreateOptions,
+  SavedObjectsBulkGetObject,
 } from 'kibana/server';
 // eslint-disable-next-line no-restricted-imports
 import { legacyRuleStatusSavedObjectType } from '../../rules/legacy_rule_status/legacy_rule_status_saved_object_mappings';
@@ -30,7 +31,8 @@ export interface RuleStatusSavedObjectsClient {
   ) => Promise<SavedObject<IRuleStatusSOAttributes>>;
   update: (
     id: string,
-    attributes: Partial<IRuleStatusSOAttributes>
+    attributes: Partial<IRuleStatusSOAttributes>,
+    options?: SavedObjectsCreateOptions
   ) => Promise<SavedObjectsUpdateResponse<IRuleStatusSOAttributes>>;
   delete: (id: string) => Promise<{}>;
 }
@@ -56,53 +58,78 @@ export const ruleStatusSavedObjectsClientFactory = (
     if (ids.length === 0) {
       return {};
     }
+    // With migration from `alertId` to `references[].id` it's not possible to fetch
+    // just the most recent RuleStatusSO's in one query as SO.find() API doesn't support
+    // `reverse_nested` so you can't include the parent. Broken out into two queries,
+    // first for fetching most recent RuleStatusSO id's, then the objects themself.
+    // TODO: Still use one query but return all status SO's and filter server side? Perf test?
+
+    // Query 1: Fetch most recent RuleStatusSO _id's
     const references = ids.map<SavedObjectsFindOptionsReference>((alertId) => ({
       id: alertId,
       type: 'alert',
     }));
     const order: 'desc' = 'desc';
-    const aggs = {
-      alertIds: {
-        terms: {
-          field: `${legacyRuleStatusSavedObjectType}.references.id`,
-          size: ids.length,
+    const nestedAggs = {
+      references: {
+        nested: {
+          path: `${legacyRuleStatusSavedObjectType}.references`,
         },
         aggs: {
-          most_recent_statuses: {
-            top_hits: {
-              sort: [
-                {
-                  [`${legacyRuleStatusSavedObjectType}.statusDate`]: {
-                    order,
-                  },
+          alertIds: {
+            terms: {
+              field: `${legacyRuleStatusSavedObjectType}.references.id`,
+              size: ids.length,
+            },
+            aggs: {
+              most_recent_statuses: {
+                top_hits: {
+                  sort: [
+                    {
+                      [`${legacyRuleStatusSavedObjectType}.statusDate`]: {
+                        order,
+                      },
+                    },
+                  ],
+                  size: statusesPerId,
                 },
-              ],
-              size: statusesPerId,
+              },
             },
           },
         },
       },
     };
-    const results = await savedObjectsClient.find({
+    const statusIdResults = await savedObjectsClient.find({
       hasReference: references,
-      aggs,
+      aggs: nestedAggs,
       type: legacyRuleStatusSavedObjectType,
       perPage: 0,
     });
-    const buckets = get(results, 'aggregations.alertIds.buckets');
-    return buckets.reduce((acc: Record<string, unknown>, bucket: unknown) => {
-      const key = get(bucket, 'key');
-      const hits = get(bucket, 'most_recent_statuses.hits.hits');
-      // eslint-disable-next-line @typescript-eslint/no-explicit-any
-      const statuses = hits.map((hit: any) => hit._source['siem-detection-engine-rule-status']);
-      acc[key] = statuses;
+    const statusIdResultBuckets = get(statusIdResults, 'aggregations.references.alertIds.buckets');
+    const ruleStatusIds: string[] = statusIdResultBuckets.map((b: unknown) => {
+      const hits = get(b, 'most_recent_statuses.hits.hits');
+      return get(hits, `[${hits.length - 1}]._id`); // TODO: top_hits agg doesn't appear to be working above
+    });
+
+    // Query 2: Retrieve RuleStatusSO objects via `_id`'s
+    const objects: SavedObjectsBulkGetObject[] = ruleStatusIds.map((id) => ({
+      type: legacyRuleStatusSavedObjectType,
+      id: id.substring(`${legacyRuleStatusSavedObjectType}:`.length), // TODO: Any gotchas here w/ any potential additional share-capable prefixes
+    }));
+    const statusResult = await savedObjectsClient.bulkGet(objects);
+
+    const hits = get(statusResult, 'saved_objects');
+    return hits.reduce((acc: Record<string, IRuleStatusSOAttributes[]>, hit: unknown) => {
+      const alertId: string = get(hit, `references[0].id`);
+      // TODO: Fine to not support multi-status now?
+      acc[alertId] = [get(hit, 'attributes')];
       return acc;
     }, {});
   },
   create: (attributes, options) => {
     return savedObjectsClient.create(legacyRuleStatusSavedObjectType, attributes, options);
   },
-  update: (id, attributes) =>
-    savedObjectsClient.update(legacyRuleStatusSavedObjectType, id, attributes),
+  update: (id, attributes, options) =>
+    savedObjectsClient.update(legacyRuleStatusSavedObjectType, id, attributes, options),
   delete: (id) => savedObjectsClient.delete(legacyRuleStatusSavedObjectType, id),
 });

x-pack/plugins/security_solution/server/lib/detection_engine/rule_execution_log/saved_objects_adapter/saved_objects_adapter.ts

@@ -54,7 +54,7 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient {
       sortField: 'statusDate',
       sortOrder: 'desc',
       search: ruleId,
-      searchFields: ['alertId'],
+      searchFields: ['references.id'],
     });
   }
 
@@ -71,12 +71,17 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient {
   }
 
   public async logExecutionMetrics({ ruleId, metrics }: LogExecutionMetricsArgs) {
+    const references: SavedObjectReference[] = [legacyGetRuleReference(ruleId)];
     const [currentStatus] = await this.getOrCreateRuleStatuses(ruleId);
 
-    await this.ruleStatusClient.update(currentStatus.id, {
-      ...currentStatus.attributes,
-      ...convertMetricFields(metrics),
-    });
+    await this.ruleStatusClient.update(
+      currentStatus.id,
+      {
+        ...currentStatus.attributes,
+        ...convertMetricFields(metrics),
+      },
+      { references }
+    );
   }
 
   private createNewRuleStatus = async (
@@ -86,7 +91,6 @@ export class SavedObjectsAdapter implements IRuleExecutionLogClient {
     const now = new Date().toISOString();
     return this.ruleStatusClient.create(
       {
-        alertId: ruleId,
         statusDate: now,
         status: RuleExecutionStatus['going to run'],
         lastFailureAt: null,

x-pack/plugins/security_solution/server/lib/detection_engine/rules/types.ts

@@ -111,7 +111,6 @@ export type RuleAlertType = SanitizedAlert<RuleParams>;
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any
 export interface IRuleStatusSOAttributes extends Record<string, any> {
-  alertId: string; // created alert id. // TODO: Remove?
   statusDate: StatusDate;
   lastFailureAt: LastFailureAt | null | undefined;
   lastFailureMessage: LastFailureMessage | null | undefined;