[Alerting] formalize alert status and add status fields to alert save…

…d object (elastic#75553)

resolves elastic#51099

This formalizes the concept of "alert status", in terms of it's execution, with
some new fields in the alert saved object and types used with the alert client
and http APIs.

These fields are read-only from the client point-of-view; they are provided in
the alert structures, but are only updated by the alerting framework itself.
The values will be updated after each run of the alert type executor.

The data is added to the alert as the `executionStatus` field, with the
following shape:

```ts
interface AlertExecutionStatus {
  status: 'ok' | 'active' | 'error' | 'pending' | 'unknown';
  lastExecutionDate: Date;
  error?: {
    reason: 'read' | 'decrypt' | 'execute' | 'unknown';
    message: string;
  };
}
```

x-pack/plugins/alerts/common/alert.ts

@@ -15,6 +15,28 @@ export interface IntervalSchedule extends SavedObjectAttributes {
   interval: string;
 }
 
+// for the `typeof ThingValues[number]` types below, become string types that
+// only accept the values in the associated string arrays
+export const AlertExecutionStatusValues = ['ok', 'active', 'error', 'pending', 'unknown'] as const;
+export type AlertExecutionStatuses = typeof AlertExecutionStatusValues[number];
+
+export const AlertExecutionStatusErrorReasonValues = [
+  'read',
+  'decrypt',
+  'execute',
+  'unknown',
+] as const;
+export type AlertExecutionStatusErrorReasons = typeof AlertExecutionStatusErrorReasonValues[number];
+
+export interface AlertExecutionStatus {
+  status: AlertExecutionStatuses;
+  lastExecutionDate: Date;
+  error?: {
+    reason: AlertExecutionStatusErrorReasons;
+    message: string;
+  };
+}
+
 export type AlertActionParams = SavedObjectAttributes;
 
 export interface AlertAction {
@@ -44,6 +66,7 @@ export interface Alert {
   throttle: string | null;
   muteAll: boolean;
   mutedInstanceIds: string[];
+  executionStatus: AlertExecutionStatus;
 }
 
 export type SanitizedAlert = Omit<Alert, 'apiKey'>;

x-pack/plugins/alerts/server/alerts_client.test.ts

@@ -393,6 +393,11 @@ describe('create()', () => {
         "createdAt": "2019-02-12T21:01:22.479Z",
         "createdBy": "elastic",
         "enabled": true,
+        "executionStatus": Object {
+          "error": null,
+          "lastExecutionDate": "2019-02-12T21:01:22.479Z",
+          "status": "pending",
+        },
         "meta": Object {
           "versionApiKeyLastmodified": "v7.10.0",
         },
@@ -1034,6 +1039,11 @@ describe('create()', () => {
         muteAll: false,
         mutedInstanceIds: [],
         tags: ['foo'],
+        executionStatus: {
+          lastExecutionDate: '2019-02-12T21:01:22.479Z',
+          status: 'pending',
+          error: null,
+        },
       },
       {
         references: [
@@ -1150,6 +1160,11 @@ describe('create()', () => {
         muteAll: false,
         mutedInstanceIds: [],
         tags: ['foo'],
+        executionStatus: {
+          lastExecutionDate: '2019-02-12T21:01:22.479Z',
+          status: 'pending',
+          error: null,
+        },
       },
       {
         references: [
@@ -2506,6 +2521,11 @@ const BaseAlertInstanceSummarySavedObject: SavedObject<RawAlert> = {
     throttle: null,
     muteAll: false,
     mutedInstanceIds: [],
+    executionStatus: {
+      status: 'unknown',
+      lastExecutionDate: '2020-08-20T19:23:38Z',
+      error: null,
+    },
   },
   references: [],
 };

x-pack/plugins/alerts/server/alerts_client.ts

@@ -28,7 +28,7 @@ import {
   AlertTaskState,
   AlertInstanceSummary,
 } from './types';
-import { validateAlertTypeParams } from './lib';
+import { validateAlertTypeParams, alertExecutionStatusFromRaw } from './lib';
 import {
   InvalidateAPIKeyParams,
   GrantAPIKeyResult as SecurityPluginGrantAPIKeyResult,
@@ -122,6 +122,7 @@ export interface CreateOptions {
     | 'muteAll'
     | 'mutedInstanceIds'
     | 'actions'
+    | 'executionStatus'
   > & { actions: NormalizedAlertAction[] };
   options?: {
     migrationVersion?: Record<string, string>;
@@ -228,6 +229,11 @@ export class AlertsClient {
       params: validatedAlertTypeParams as RawAlert['params'],
       muteAll: false,
       mutedInstanceIds: [],
+      executionStatus: {
+        status: 'pending',
+        lastExecutionDate: new Date().toISOString(),
+        error: null,
+      },
     };
     let createdAlert: SavedObject<RawAlert>;
     try {
@@ -978,9 +984,19 @@ export class AlertsClient {
     updatedAt: SavedObject['updated_at'] = createdAt,
     references: SavedObjectReference[] | undefined
   ): PartialAlert {
+    // Not the prettiest code here, but if we want to use most of the
+    // alert fields from the rawAlert using `...rawAlert` kind of access, we
+    // need to specifically delete the executionStatus as it's a different type
+    // in RawAlert and Alert.  Probably next time we need to do something similar
+    // here, we should look at redesigning the implementation of this method.
+    const rawAlertWithoutExecutionStatus: Partial<Omit<RawAlert, 'executionStatus'>> = {
+      ...rawAlert,
+    };
+    delete rawAlertWithoutExecutionStatus.executionStatus;
+    const executionStatus = alertExecutionStatusFromRaw(this.logger, id, rawAlert.executionStatus);
     return {
       id,
-      ...rawAlert,
+      ...rawAlertWithoutExecutionStatus,
       // we currently only support the Interval Schedule type
       // Once we support additional types, this type signature will likely change
       schedule: rawAlert.schedule as IntervalSchedule,
@@ -990,6 +1006,7 @@ export class AlertsClient {
       ...(updatedAt ? { updatedAt: new Date(updatedAt) } : {}),
       ...(createdAt ? { createdAt: new Date(createdAt) } : {}),
       ...(scheduledTaskId ? { scheduledTaskId } : {}),
+      ...(executionStatus ? { executionStatus } : {}),
     };
   }
 

x-pack/plugins/alerts/server/lib/alert_execution_status.test.ts

@@ -0,0 +1,185 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { loggingSystemMock } from '../../../../../src/core/server/mocks';
+import { AlertExecutionStatusErrorReasons } from '../types';
+import {
+  executionStatusFromState,
+  executionStatusFromError,
+  alertExecutionStatusToRaw,
+  alertExecutionStatusFromRaw,
+} from './alert_execution_status';
+import { ErrorWithReason } from './error_with_reason';
+
+const MockLogger = loggingSystemMock.create().get();
+
+describe('AlertExecutionStatus', () => {
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  describe('executionStatusFromState()', () => {
+    test('empty task state', () => {
+      const status = executionStatusFromState({});
+      checkDateIsNearNow(status.lastExecutionDate);
+      expect(status.status).toBe('ok');
+      expect(status.error).toBe(undefined);
+    });
+
+    test('task state with no instances', () => {
+      const status = executionStatusFromState({ alertInstances: {} });
+      checkDateIsNearNow(status.lastExecutionDate);
+      expect(status.status).toBe('ok');
+      expect(status.error).toBe(undefined);
+    });
+
+    test('task state with one instance', () => {
+      const status = executionStatusFromState({ alertInstances: { a: {} } });
+      checkDateIsNearNow(status.lastExecutionDate);
+      expect(status.status).toBe('active');
+      expect(status.error).toBe(undefined);
+    });
+  });
+
+  describe('executionStatusFromError()', () => {
+    test('error with no reason', () => {
+      const status = executionStatusFromError(new Error('boo!'));
+      expect(status.status).toBe('error');
+      expect(status.error).toMatchInlineSnapshot(`
+        Object {
+          "message": "boo!",
+          "reason": "unknown",
+        }
+      `);
+    });
+
+    test('error with a reason', () => {
+      const status = executionStatusFromError(new ErrorWithReason('execute', new Error('hoo!')));
+      expect(status.status).toBe('error');
+      expect(status.error).toMatchInlineSnapshot(`
+        Object {
+          "message": "hoo!",
+          "reason": "execute",
+        }
+      `);
+    });
+  });
+
+  describe('alertExecutionStatusToRaw()', () => {
+    const date = new Date('2020-09-03T16:26:58Z');
+    const status = 'ok';
+    const reason: AlertExecutionStatusErrorReasons = 'decrypt';
+    const error = { reason, message: 'wops' };
+
+    test('status without an error', () => {
+      expect(alertExecutionStatusToRaw({ lastExecutionDate: date, status })).toMatchInlineSnapshot(`
+        Object {
+          "error": null,
+          "lastExecutionDate": "2020-09-03T16:26:58.000Z",
+          "status": "ok",
+        }
+      `);
+    });
+
+    test('status with an error', () => {
+      expect(alertExecutionStatusToRaw({ lastExecutionDate: date, status, error }))
+        .toMatchInlineSnapshot(`
+        Object {
+          "error": Object {
+            "message": "wops",
+            "reason": "decrypt",
+          },
+          "lastExecutionDate": "2020-09-03T16:26:58.000Z",
+          "status": "ok",
+        }
+      `);
+    });
+  });
+
+  describe('alertExecutionStatusFromRaw()', () => {
+    const date = new Date('2020-09-03T16:26:58Z').toISOString();
+    const status = 'active';
+    const reason: AlertExecutionStatusErrorReasons = 'execute';
+    const error = { reason, message: 'wops' };
+
+    test('no input', () => {
+      const result = alertExecutionStatusFromRaw(MockLogger, 'alert-id');
+      expect(result).toBe(undefined);
+    });
+
+    test('undefined input', () => {
+      const result = alertExecutionStatusFromRaw(MockLogger, 'alert-id', undefined);
+      expect(result).toBe(undefined);
+    });
+
+    test('null input', () => {
+      const result = alertExecutionStatusFromRaw(MockLogger, 'alert-id', null);
+      expect(result).toBe(undefined);
+    });
+
+    test('invalid date', () => {
+      const result = alertExecutionStatusFromRaw(MockLogger, 'alert-id', {
+        lastExecutionDate: 'an invalid date',
+      })!;
+      checkDateIsNearNow(result.lastExecutionDate);
+      expect(result.status).toBe('unknown');
+      expect(result.error).toBe(undefined);
+      expect(MockLogger.debug).toBeCalledWith(
+        'invalid alertExecutionStatus lastExecutionDate "an invalid date" in raw alert alert-id'
+      );
+    });
+
+    test('valid date', () => {
+      const result = alertExecutionStatusFromRaw(MockLogger, 'alert-id', {
+        lastExecutionDate: date,
+      });
+      expect(result).toMatchInlineSnapshot(`
+        Object {
+          "lastExecutionDate": 2020-09-03T16:26:58.000Z,
+          "status": "unknown",
+        }
+      `);
+    });
+
+    test('valid status and date', () => {
+      const result = alertExecutionStatusFromRaw(MockLogger, 'alert-id', {
+        status,
+        lastExecutionDate: date,
+      });
+      expect(result).toMatchInlineSnapshot(`
+        Object {
+          "lastExecutionDate": 2020-09-03T16:26:58.000Z,
+          "status": "active",
+        }
+      `);
+    });
+
+    test('valid status, date and error', () => {
+      const result = alertExecutionStatusFromRaw(MockLogger, 'alert-id', {
+        status,
+        lastExecutionDate: date,
+        error,
+      });
+      expect(result).toMatchInlineSnapshot(`
+        Object {
+          "error": Object {
+            "message": "wops",
+            "reason": "execute",
+          },
+          "lastExecutionDate": 2020-09-03T16:26:58.000Z,
+          "status": "active",
+        }
+      `);
+    });
+  });
+});
+
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+function checkDateIsNearNow(date: any) {
+  expect(date instanceof Date).toBe(true);
+  // allow for lots of slop in the time difference
+  expect(Date.now() - date.valueOf()).toBeLessThanOrEqual(10000);
+}

x-pack/plugins/alerts/server/lib/alert_execution_status.ts

@@ -0,0 +1,66 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import { Logger } from 'src/core/server';
+import { AlertTaskState, AlertExecutionStatus, RawAlertExecutionStatus } from '../types';
+import { getReasonFromError } from './error_with_reason';
+
+export function executionStatusFromState(state: AlertTaskState): AlertExecutionStatus {
+  const instanceIds = Object.keys(state.alertInstances ?? {});
+  return {
+    lastExecutionDate: new Date(),
+    status: instanceIds.length === 0 ? 'ok' : 'active',
+  };
+}
+
+export function executionStatusFromError(error: Error): AlertExecutionStatus {
+  return {
+    lastExecutionDate: new Date(),
+    status: 'error',
+    error: {
+      reason: getReasonFromError(error),
+      message: error.message,
+    },
+  };
+}
+
+export function alertExecutionStatusToRaw({
+  lastExecutionDate,
+  status,
+  error,
+}: AlertExecutionStatus): RawAlertExecutionStatus {
+  return {
+    lastExecutionDate: lastExecutionDate.toISOString(),
+    status,
+    // explicitly setting to null (in case undefined) due to partial update concerns
+    error: error ?? null,
+  };
+}
+
+export function alertExecutionStatusFromRaw(
+  logger: Logger,
+  alertId: string,
+  rawAlertExecutionStatus?: Partial<RawAlertExecutionStatus> | null | undefined
+): AlertExecutionStatus | undefined {
+  if (!rawAlertExecutionStatus) return undefined;
+
+  const { lastExecutionDate, status = 'unknown', error } = rawAlertExecutionStatus;
+
+  let parsedDateMillis = lastExecutionDate ? Date.parse(lastExecutionDate) : Date.now();
+  if (isNaN(parsedDateMillis)) {
+    logger.debug(
+      `invalid alertExecutionStatus lastExecutionDate "${lastExecutionDate}" in raw alert ${alertId}`
+    );
+    parsedDateMillis = Date.now();
+  }
+
+  const parsedDate = new Date(parsedDateMillis);
+  if (error) {
+    return { lastExecutionDate: parsedDate, status, error };
+  } else {
+    return { lastExecutionDate: parsedDate, status };
+  }
+}