diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 40677dc5a498e6..f44a2e962f1fa8 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -19,6 +19,7 @@ x-pack/examples/alerting_example @elastic/response-ops x-pack/test/functional_with_es_ssl/plugins/alerts @elastic/response-ops x-pack/plugins/alerting @elastic/response-ops packages/kbn-alerts @elastic/security-solution +packages/kbn-alerts-as-data-utils @elastic/response-ops x-pack/test/alerting_api_integration/common/plugins/alerts_restricted @elastic/response-ops packages/kbn-alerts-ui-shared @elastic/response-ops packages/kbn-ambient-common-types @elastic/kibana-operations diff --git a/package.json b/package.json index fd551a6571e62b..7f1f7db7631398 100644 --- a/package.json +++ b/package.json @@ -134,6 +134,7 @@ "@kbn/alerting-fixture-plugin": "link:x-pack/test/functional_with_es_ssl/plugins/alerts", "@kbn/alerting-plugin": "link:x-pack/plugins/alerting", "@kbn/alerts": "link:packages/kbn-alerts", + "@kbn/alerts-as-data-utils": "link:packages/kbn-alerts-as-data-utils", "@kbn/alerts-restricted-fixtures-plugin": "link:x-pack/test/alerting_api_integration/common/plugins/alerts_restricted", "@kbn/alerts-ui-shared": "link:packages/kbn-alerts-ui-shared", "@kbn/analytics": "link:packages/kbn-analytics", diff --git a/packages/kbn-alerts-as-data-utils/index.ts b/packages/kbn-alerts-as-data-utils/index.ts new file mode 100644 index 00000000000000..7b2cad2ca54404 --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/index.ts @@ -0,0 +1,9 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './src/field_maps'; diff --git a/packages/kbn-alerts-as-data-utils/kibana.jsonc b/packages/kbn-alerts-as-data-utils/kibana.jsonc new file mode 100644 index 00000000000000..07e8490dde7b53 --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/kibana.jsonc @@ -0,0 +1,5 @@ +{ + "type": "shared-common", + "id": "@kbn/alerts-as-data-utils", + "owner": "@elastic/response-ops" +} diff --git a/packages/kbn-alerts-as-data-utils/package.json b/packages/kbn-alerts-as-data-utils/package.json new file mode 100644 index 00000000000000..25aa26b3d435c4 --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/package.json @@ -0,0 +1,6 @@ +{ + "name": "@kbn/alerts-as-data-utils", + "private": true, + "version": "1.0.0", + "license": "SSPL-1.0 OR Elastic License 2.0" +} \ No newline at end of file diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/alert_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts similarity index 77% rename from x-pack/plugins/alerting/common/alert_schema/field_maps/alert_field_map.ts rename to packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts index 4613415e0fa002..8e5a606a910176 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/alert_field_map.ts +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/alert_field_map.ts @@ -1,16 +1,20 @@ /* * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ import { ALERT_ACTION_GROUP, + ALERT_CASE_IDS, ALERT_DURATION, ALERT_END, ALERT_FLAPPING, - ALERT_ID, + ALERT_FLAPPING_HISTORY, + ALERT_INSTANCE_ID, + ALERT_LAST_DETECTED, ALERT_REASON, ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, @@ -27,92 +31,93 @@ import { ALERT_UUID, ALERT_WORKFLOW_STATUS, SPACE_IDS, + TIMESTAMP, VERSION, } from '@kbn/rule-data-utils'; export const alertFieldMap = { - [ALERT_RULE_PARAMETERS]: { - type: 'object', - enabled: false, + [ALERT_ACTION_GROUP]: { + type: 'keyword', + array: false, required: false, }, - [ALERT_RULE_TYPE_ID]: { + [ALERT_CASE_IDS]: { type: 'keyword', + array: true, + required: false, + }, + [ALERT_DURATION]: { + type: 'long', array: false, - required: true, + required: false, }, - [ALERT_RULE_CONSUMER]: { - type: 'keyword', + [ALERT_END]: { + type: 'date', array: false, - required: true, + required: false, }, - [ALERT_RULE_PRODUCER]: { - type: 'keyword', + [ALERT_FLAPPING]: { + type: 'boolean', array: false, - required: true, + required: false, }, - [SPACE_IDS]: { - type: 'keyword', + [ALERT_FLAPPING_HISTORY]: { + type: 'boolean', array: true, - required: true, - }, - [ALERT_UUID]: { - type: 'keyword', - array: false, - required: true, + required: false, }, - [ALERT_ID]: { + [ALERT_INSTANCE_ID]: { type: 'keyword', array: false, required: true, }, - [ALERT_START]: { + [ALERT_LAST_DETECTED]: { type: 'date', - array: false, required: false, - }, - [ALERT_TIME_RANGE]: { - type: 'date_range', - format: 'epoch_millis||strict_date_optional_time', array: false, - required: false, }, - [ALERT_END]: { - type: 'date', + [ALERT_REASON]: { + type: 'keyword', array: false, required: false, }, - [ALERT_DURATION]: { - type: 'long', + [ALERT_RULE_CATEGORY]: { + type: 'keyword', array: false, - required: false, + required: true, }, - [ALERT_STATUS]: { + [ALERT_RULE_CONSUMER]: { type: 'keyword', array: false, required: true, }, - [VERSION]: { - type: 'version', + [ALERT_RULE_EXECUTION_UUID]: { + type: 'keyword', array: false, required: false, }, - [ALERT_WORKFLOW_STATUS]: { + [ALERT_RULE_NAME]: { type: 'keyword', array: false, - required: false, + required: true, }, - [ALERT_ACTION_GROUP]: { - type: 'keyword', + [ALERT_RULE_PARAMETERS]: { array: false, + type: 'flattened', + ignore_above: 4096, required: false, }, - [ALERT_REASON]: { + [ALERT_RULE_PRODUCER]: { type: 'keyword', array: false, + required: true, + }, + [ALERT_RULE_TAGS]: { + type: 'keyword', + array: true, required: false, }, - [ALERT_RULE_CATEGORY]: { + [ALERT_RULE_TYPE_ID]: { type: 'keyword', array: false, required: true, @@ -122,26 +127,47 @@ export const alertFieldMap = { array: false, required: true, }, - [ALERT_RULE_EXECUTION_UUID]: { + [ALERT_START]: { + type: 'date', + array: false, + required: false, + }, + [ALERT_STATUS]: { type: 'keyword', array: false, + required: true, + }, + [ALERT_TIME_RANGE]: { + type: 'date_range', + format: 'epoch_millis||strict_date_optional_time', + array: false, required: false, }, - [ALERT_RULE_NAME]: { + [ALERT_UUID]: { type: 'keyword', array: false, required: true, }, - [ALERT_RULE_TAGS]: { + [ALERT_WORKFLOW_STATUS]: { type: 'keyword', - array: true, + array: false, required: false, }, - [ALERT_FLAPPING]: { - type: 'boolean', + [SPACE_IDS]: { + type: 'keyword', + array: true, + required: true, + }, + [TIMESTAMP]: { + type: 'date', + required: true, + array: false, + }, + [VERSION]: { + type: 'version', array: false, required: false, }, -}; +} as const; export type AlertFieldMap = typeof alertFieldMap; diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts new file mode 100644 index 00000000000000..9294a12b4ce508 --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/ecs_field_map.ts @@ -0,0 +1,27 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { EcsFlat } from '@kbn/ecs'; +import { EcsMetadata, FieldMap } from './types'; + +export const ecsFieldMap: FieldMap = Object.keys(EcsFlat).reduce((acc, currKey) => { + const value: EcsMetadata = EcsFlat[currKey as keyof typeof EcsFlat]; + return { + ...acc, + [currKey]: { + type: value.type, + array: value.normalize.includes('array'), + required: !!value.required, + ...(value.scaling_factor ? { scaling_factor: value.scaling_factor } : {}), + ...(value.ignore_above ? { ignore_above: value.ignore_above } : {}), + ...(value.multi_fields ? { multi_fields: value.multi_fields } : {}), + }, + }; +}, {}); + +export type EcsFieldMap = typeof ecsFieldMap; diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/index.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/index.ts new file mode 100644 index 00000000000000..9aef7690b343ca --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/index.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export * from './alert_field_map'; +export * from './ecs_field_map'; +export * from './legacy_alert_field_map'; +export type { FieldMap, MultiField } from './types'; diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/legacy_alert_field_map.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/legacy_alert_field_map.ts new file mode 100644 index 00000000000000..6faa403188fdbe --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/legacy_alert_field_map.ts @@ -0,0 +1,202 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { + ALERT_RISK_SCORE, + ALERT_RULE_AUTHOR, + ALERT_RULE_CREATED_AT, + ALERT_RULE_CREATED_BY, + ALERT_RULE_DESCRIPTION, + ALERT_RULE_ENABLED, + ALERT_RULE_FROM, + ALERT_RULE_INTERVAL, + ALERT_RULE_LICENSE, + ALERT_RULE_NOTE, + ALERT_RULE_REFERENCES, + ALERT_RULE_RULE_ID, + ALERT_RULE_RULE_NAME_OVERRIDE, + ALERT_RULE_TO, + ALERT_RULE_TYPE, + ALERT_RULE_UPDATED_AT, + ALERT_RULE_UPDATED_BY, + ALERT_RULE_VERSION, + ALERT_SEVERITY, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_FIELD, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_VALUE, + ALERT_SYSTEM_STATUS, + ALERT_WORKFLOW_REASON, + ALERT_WORKFLOW_USER, + ECS_VERSION, + EVENT_ACTION, + EVENT_KIND, + TAGS, +} from '@kbn/rule-data-utils'; + +export const legacyAlertFieldMap = { + [ALERT_RISK_SCORE]: { + type: 'float', + array: false, + required: false, + }, + [ALERT_RULE_AUTHOR]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_CREATED_AT]: { + type: 'date', + array: false, + required: false, + }, + [ALERT_RULE_CREATED_BY]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_DESCRIPTION]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_ENABLED]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_FROM]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_INTERVAL]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_LICENSE]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_NOTE]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_REFERENCES]: { + type: 'keyword', + array: true, + required: false, + }, + [ALERT_RULE_RULE_ID]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_RULE_NAME_OVERRIDE]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_TO]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_TYPE]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_UPDATED_AT]: { + type: 'date', + array: false, + required: false, + }, + [ALERT_RULE_UPDATED_BY]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_RULE_VERSION]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_SEVERITY]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_SUPPRESSION_DOCS_COUNT]: { + type: 'long', + array: false, + required: false, + }, + [ALERT_SUPPRESSION_END]: { + type: 'date', + array: false, + required: false, + }, + [ALERT_SUPPRESSION_FIELD]: { + type: 'keyword', + array: true, + required: false, + }, + [ALERT_SUPPRESSION_START]: { + type: 'date', + array: false, + required: false, + }, + [ALERT_SUPPRESSION_VALUE]: { + type: 'keyword', + array: true, + required: false, + }, + [ALERT_SYSTEM_STATUS]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_WORKFLOW_REASON]: { + type: 'keyword', + array: false, + required: false, + }, + [ALERT_WORKFLOW_USER]: { + type: 'keyword', + array: false, + required: false, + }, + // get these from ecs field map when available + [ECS_VERSION]: { + type: 'keyword', + array: false, + required: false, + }, + [EVENT_ACTION]: { + type: 'keyword', + array: false, + required: false, + }, + [EVENT_KIND]: { + type: 'keyword', + array: false, + required: false, + }, + [TAGS]: { + type: 'keyword', + array: true, + required: false, + }, +} as const; + +export type LegacyAlertFieldMap = typeof legacyAlertFieldMap; diff --git a/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts b/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts new file mode 100644 index 00000000000000..04f9d045f6e28a --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/src/field_maps/types.ts @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +export interface AllowedValue { + description?: string; + name?: string; +} + +export interface MultiField { + flat_name: string; + name: string; + type: string; +} + +export interface EcsMetadata { + allowed_values?: AllowedValue[]; + dashed_name: string; + description: string; + doc_values?: boolean; + example?: string | number | boolean; + flat_name: string; + ignore_above?: number; + index?: boolean; + level: string; + multi_fields?: MultiField[]; + name: string; + normalize: string[]; + required?: boolean; + scaling_factor?: number; + short: string; + type: string; +} + +export interface FieldMap { + [key: string]: { + type: string; + required: boolean; + array?: boolean; + doc_values?: boolean; + enabled?: boolean; + format?: string; + ignore_above?: number; + multi_fields?: MultiField[]; + index?: boolean; + path?: string; + scaling_factor?: number; + dynamic?: boolean | 'strict'; + }; +} diff --git a/packages/kbn-alerts-as-data-utils/tsconfig.json b/packages/kbn-alerts-as-data-utils/tsconfig.json new file mode 100644 index 00000000000000..00b7ffc082c954 --- /dev/null +++ b/packages/kbn-alerts-as-data-utils/tsconfig.json @@ -0,0 +1,20 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "outDir": "target/types", + "types": [ + "jest", + "node" + ] + }, + "include": [ + "**/*.ts" + ], + "exclude": [ + "target/**/*" + ], + "kbn_references": [ + "@kbn/ecs", + "@kbn/rule-data-utils", + ] +} diff --git a/packages/kbn-rule-data-utils/index.ts b/packages/kbn-rule-data-utils/index.ts index 145cc2c5220e9f..ea0028b972ed9e 100644 --- a/packages/kbn-rule-data-utils/index.ts +++ b/packages/kbn-rule-data-utils/index.ts @@ -7,6 +7,7 @@ */ export * from './src/default_alerts_as_data'; +export * from './src/legacy_alerts_as_data'; export * from './src/technical_field_names'; export * from './src/alerts_as_data_rbac'; export * from './src/alerts_as_data_severity'; diff --git a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts index 3a982124b58e60..34b04116b95224 100644 --- a/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts +++ b/packages/kbn-rule-data-utils/src/default_alerts_as_data.ts @@ -8,6 +8,9 @@ import { ValuesType } from 'utility-types'; +const TIMESTAMP = '@timestamp' as const; + +// namespaces const KIBANA_NAMESPACE = 'kibana' as const; const ALERT_NAMESPACE = `${KIBANA_NAMESPACE}.alert` as const; const ALERT_RULE_NAMESPACE = `${ALERT_NAMESPACE}.rule` as const; @@ -21,6 +24,9 @@ const VERSION = `${KIBANA_NAMESPACE}.version` as const; // kibana.alert.action_group - framework action group ID for this alert const ALERT_ACTION_GROUP = `${ALERT_NAMESPACE}.action_group` as const; +// kibana.alert.case_ids - array of cases associated with the alert +const ALERT_CASE_IDS = `${ALERT_NAMESPACE}.case_ids` as const; + // kibana.alert.duration.us - alert duration in nanoseconds - updated each execution // that the alert is active const ALERT_DURATION = `${ALERT_NAMESPACE}.duration.us` as const; @@ -31,8 +37,11 @@ const ALERT_END = `${ALERT_NAMESPACE}.end` as const; // kibana.alert.flapping - whether the alert is currently in a flapping state const ALERT_FLAPPING = `${ALERT_NAMESPACE}.flapping` as const; -// kibana.alert.id - alert ID, also known as alert instance ID -const ALERT_ID = `${ALERT_NAMESPACE}.id` as const; +// kibana.alert.flapping_history - whether the alert is currently in a flapping state +const ALERT_FLAPPING_HISTORY = `${ALERT_NAMESPACE}.flapping_history` as const; + +// kibana.alert.instance.id - alert ID, also known as alert instance ID +const ALERT_INSTANCE_ID = `${ALERT_NAMESPACE}.instance.id` as const; // kibana.alert.last_detected - timestamp when the alert was last seen const ALERT_LAST_DETECTED = `${ALERT_NAMESPACE}.last_detected` as const; @@ -90,10 +99,12 @@ const namespaces = { const fields = { ALERT_ACTION_GROUP, + ALERT_CASE_IDS, ALERT_DURATION, ALERT_END, ALERT_FLAPPING, - ALERT_ID, + ALERT_FLAPPING_HISTORY, + ALERT_INSTANCE_ID, ALERT_LAST_DETECTED, ALERT_REASON, ALERT_RULE_CATEGORY, @@ -111,15 +122,24 @@ const fields = { ALERT_UUID, ALERT_WORKFLOW_STATUS, SPACE_IDS, + TIMESTAMP, VERSION, }; export { + // namespaces + ALERT_NAMESPACE, + ALERT_RULE_NAMESPACE, + KIBANA_NAMESPACE, + + // fields ALERT_ACTION_GROUP, + ALERT_CASE_IDS, ALERT_DURATION, ALERT_END, ALERT_FLAPPING, - ALERT_ID, + ALERT_FLAPPING_HISTORY, + ALERT_INSTANCE_ID, ALERT_LAST_DETECTED, ALERT_REASON, ALERT_RULE_CATEGORY, @@ -137,10 +157,8 @@ export { ALERT_UUID, ALERT_WORKFLOW_STATUS, SPACE_IDS, + TIMESTAMP, VERSION, - ALERT_NAMESPACE, - ALERT_RULE_NAMESPACE, - KIBANA_NAMESPACE, }; export type DefaultAlertFieldName = ValuesType; diff --git a/packages/kbn-rule-data-utils/src/legacy_alerts_as_data.ts b/packages/kbn-rule-data-utils/src/legacy_alerts_as_data.ts new file mode 100644 index 00000000000000..4dd6c2be0c2a69 --- /dev/null +++ b/packages/kbn-rule-data-utils/src/legacy_alerts_as_data.ts @@ -0,0 +1,84 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +import { ALERT_NAMESPACE, ALERT_RULE_NAMESPACE } from './default_alerts_as_data'; + +const ECS_VERSION = 'ecs.version' as const; +const EVENT_ACTION = 'event.action' as const; +const EVENT_KIND = 'event.kind' as const; +const TAGS = 'tags' as const; + +// These are the fields that are in the rule registry technical component template +// that are NOT in the framework alerts as data common component template + +// We will maintain a legacy component template that can be used by legacy +// rule registry rules with these fields. +const ALERT_RISK_SCORE = `${ALERT_NAMESPACE}.risk_score` as const; +const ALERT_RULE_AUTHOR = `${ALERT_RULE_NAMESPACE}.author` as const; +const ALERT_RULE_CREATED_AT = `${ALERT_RULE_NAMESPACE}.created_at` as const; +const ALERT_RULE_CREATED_BY = `${ALERT_RULE_NAMESPACE}.created_by` as const; +const ALERT_RULE_DESCRIPTION = `${ALERT_RULE_NAMESPACE}.description` as const; +const ALERT_RULE_ENABLED = `${ALERT_RULE_NAMESPACE}.enabled` as const; +const ALERT_RULE_FROM = `${ALERT_RULE_NAMESPACE}.from` as const; +const ALERT_RULE_INTERVAL = `${ALERT_RULE_NAMESPACE}.interval` as const; +const ALERT_RULE_LICENSE = `${ALERT_RULE_NAMESPACE}.license` as const; +const ALERT_RULE_NOTE = `${ALERT_RULE_NAMESPACE}.note` as const; +const ALERT_RULE_REFERENCES = `${ALERT_RULE_NAMESPACE}.references` as const; +const ALERT_RULE_RULE_ID = `${ALERT_RULE_NAMESPACE}.rule_id` as const; +const ALERT_RULE_RULE_NAME_OVERRIDE = `${ALERT_RULE_NAMESPACE}.rule_name_override` as const; +const ALERT_RULE_TO = `${ALERT_RULE_NAMESPACE}.to` as const; +const ALERT_RULE_TYPE = `${ALERT_RULE_NAMESPACE}.type` as const; +const ALERT_RULE_UPDATED_AT = `${ALERT_RULE_NAMESPACE}.updated_at` as const; +const ALERT_RULE_UPDATED_BY = `${ALERT_RULE_NAMESPACE}.updated_by` as const; +const ALERT_RULE_VERSION = `${ALERT_RULE_NAMESPACE}.version` as const; +const ALERT_SEVERITY = `${ALERT_NAMESPACE}.severity` as const; +const ALERT_SUPPRESSION_META = `${ALERT_NAMESPACE}.suppression` as const; +const ALERT_SUPPRESSION_TERMS = `${ALERT_SUPPRESSION_META}.terms` as const; +const ALERT_SUPPRESSION_FIELD = `${ALERT_SUPPRESSION_TERMS}.field` as const; +const ALERT_SUPPRESSION_VALUE = `${ALERT_SUPPRESSION_TERMS}.value` as const; +const ALERT_SUPPRESSION_START = `${ALERT_SUPPRESSION_META}.start` as const; +const ALERT_SUPPRESSION_END = `${ALERT_SUPPRESSION_META}.end` as const; +const ALERT_SUPPRESSION_DOCS_COUNT = `${ALERT_SUPPRESSION_META}.docs_count` as const; +const ALERT_SYSTEM_STATUS = `${ALERT_NAMESPACE}.system_status` as const; +const ALERT_WORKFLOW_REASON = `${ALERT_NAMESPACE}.workflow_reason` as const; +const ALERT_WORKFLOW_USER = `${ALERT_NAMESPACE}.workflow_user` as const; + +export { + ALERT_RISK_SCORE, + ALERT_RULE_AUTHOR, + ALERT_RULE_CREATED_AT, + ALERT_RULE_CREATED_BY, + ALERT_RULE_DESCRIPTION, + ALERT_RULE_ENABLED, + ALERT_RULE_FROM, + ALERT_RULE_INTERVAL, + ALERT_RULE_LICENSE, + ALERT_RULE_NOTE, + ALERT_RULE_REFERENCES, + ALERT_RULE_RULE_ID, + ALERT_RULE_RULE_NAME_OVERRIDE, + ALERT_RULE_TO, + ALERT_RULE_TYPE, + ALERT_RULE_UPDATED_AT, + ALERT_RULE_UPDATED_BY, + ALERT_RULE_VERSION, + ALERT_SEVERITY, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_FIELD, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_VALUE, + ALERT_SYSTEM_STATUS, + ALERT_WORKFLOW_REASON, + ALERT_WORKFLOW_USER, + ECS_VERSION, + EVENT_ACTION, + EVENT_KIND, + TAGS, +}; diff --git a/packages/kbn-rule-data-utils/src/technical_field_names.ts b/packages/kbn-rule-data-utils/src/technical_field_names.ts index 89eca0f9230464..cf45162b208538 100644 --- a/packages/kbn-rule-data-utils/src/technical_field_names.ts +++ b/packages/kbn-rule-data-utils/src/technical_field_names.ts @@ -8,11 +8,15 @@ import { ValuesType } from 'utility-types'; import { + ALERT_NAMESPACE, + ALERT_RULE_NAMESPACE, KIBANA_NAMESPACE, ALERT_ACTION_GROUP, + ALERT_CASE_IDS, ALERT_DURATION, ALERT_END, ALERT_FLAPPING, + ALERT_INSTANCE_ID, ALERT_REASON, ALERT_RULE_CATEGORY, ALERT_RULE_CONSUMER, @@ -29,61 +33,61 @@ import { ALERT_UUID, ALERT_WORKFLOW_STATUS, SPACE_IDS, + TIMESTAMP, VERSION, - ALERT_NAMESPACE, - ALERT_RULE_NAMESPACE, } from './default_alerts_as_data'; +import { + ALERT_RISK_SCORE, + ALERT_RULE_AUTHOR, + ALERT_RULE_CREATED_AT, + ALERT_RULE_CREATED_BY, + ALERT_RULE_DESCRIPTION, + ALERT_RULE_ENABLED, + ALERT_RULE_FROM, + ALERT_RULE_INTERVAL, + ALERT_RULE_LICENSE, + ALERT_RULE_NOTE, + ALERT_RULE_REFERENCES, + ALERT_RULE_RULE_ID, + ALERT_RULE_RULE_NAME_OVERRIDE, + ALERT_RULE_TO, + ALERT_RULE_TYPE, + ALERT_RULE_UPDATED_AT, + ALERT_RULE_UPDATED_BY, + ALERT_RULE_VERSION, + ALERT_SEVERITY, + ALERT_SUPPRESSION_DOCS_COUNT, + ALERT_SUPPRESSION_END, + ALERT_SUPPRESSION_FIELD, + ALERT_SUPPRESSION_START, + ALERT_SUPPRESSION_TERMS, + ALERT_SUPPRESSION_VALUE, + ALERT_SYSTEM_STATUS, + ALERT_WORKFLOW_REASON, + ALERT_WORKFLOW_USER, + ECS_VERSION, + EVENT_ACTION, + EVENT_KIND, + TAGS, +} from './legacy_alerts_as_data'; + +// The following fields were identified as technical field names but were not defined in the +// rule registry technical component template. We will leave these here for backwards +// compatibility but these consts should be moved to the plugin that uses them + const ALERT_RULE_THREAT_NAMESPACE = `${ALERT_RULE_NAMESPACE}.threat` as const; -const ECS_VERSION = 'ecs.version' as const; -const EVENT_ACTION = 'event.action' as const; -const EVENT_KIND = 'event.kind' as const; const EVENT_MODULE = 'event.module' as const; -const TAGS = 'tags' as const; -const TIMESTAMP = '@timestamp' as const; // Fields pertaining to the alert const ALERT_BUILDING_BLOCK_TYPE = `${ALERT_NAMESPACE}.building_block_type` as const; const ALERT_EVALUATION_THRESHOLD = `${ALERT_NAMESPACE}.evaluation.threshold` as const; const ALERT_EVALUATION_VALUE = `${ALERT_NAMESPACE}.evaluation.value` as const; -const ALERT_INSTANCE_ID = `${ALERT_NAMESPACE}.instance.id` as const; -const ALERT_RISK_SCORE = `${ALERT_NAMESPACE}.risk_score` as const; -const ALERT_SEVERITY = `${ALERT_NAMESPACE}.severity` as const; -const ALERT_SYSTEM_STATUS = `${ALERT_NAMESPACE}.system_status` as const; -const ALERT_WORKFLOW_REASON = `${ALERT_NAMESPACE}.workflow_reason` as const; -const ALERT_WORKFLOW_USER = `${ALERT_NAMESPACE}.workflow_user` as const; -const ALERT_SUPPRESSION_META = `${ALERT_NAMESPACE}.suppression` as const; -const ALERT_SUPPRESSION_TERMS = `${ALERT_SUPPRESSION_META}.terms` as const; -const ALERT_SUPPRESSION_FIELD = `${ALERT_SUPPRESSION_TERMS}.field` as const; -const ALERT_SUPPRESSION_VALUE = `${ALERT_SUPPRESSION_TERMS}.value` as const; -const ALERT_SUPPRESSION_START = `${ALERT_SUPPRESSION_META}.start` as const; -const ALERT_SUPPRESSION_END = `${ALERT_SUPPRESSION_META}.end` as const; -const ALERT_SUPPRESSION_DOCS_COUNT = `${ALERT_SUPPRESSION_META}.docs_count` as const; - -// Fields pertaining to the cases associated with the alert -const ALERT_CASE_IDS = `${ALERT_NAMESPACE}.case_ids` as const; // Fields pertaining to the rule associated with the alert -const ALERT_RULE_AUTHOR = `${ALERT_RULE_NAMESPACE}.author` as const; -const ALERT_RULE_CREATED_AT = `${ALERT_RULE_NAMESPACE}.created_at` as const; -const ALERT_RULE_CREATED_BY = `${ALERT_RULE_NAMESPACE}.created_by` as const; -const ALERT_RULE_DESCRIPTION = `${ALERT_RULE_NAMESPACE}.description` as const; -const ALERT_RULE_ENABLED = `${ALERT_RULE_NAMESPACE}.enabled` as const; const ALERT_RULE_EXCEPTIONS_LIST = `${ALERT_RULE_NAMESPACE}.exceptions_list` as const; -const ALERT_RULE_FROM = `${ALERT_RULE_NAMESPACE}.from` as const; -const ALERT_RULE_INTERVAL = `${ALERT_RULE_NAMESPACE}.interval` as const; -const ALERT_RULE_LICENSE = `${ALERT_RULE_NAMESPACE}.license` as const; const ALERT_RULE_NAMESPACE_FIELD = `${ALERT_RULE_NAMESPACE}.namespace` as const; -const ALERT_RULE_NOTE = `${ALERT_RULE_NAMESPACE}.note` as const; -const ALERT_RULE_REFERENCES = `${ALERT_RULE_NAMESPACE}.references` as const; -const ALERT_RULE_RULE_ID = `${ALERT_RULE_NAMESPACE}.rule_id` as const; -const ALERT_RULE_RULE_NAME_OVERRIDE = `${ALERT_RULE_NAMESPACE}.rule_name_override` as const; -const ALERT_RULE_TO = `${ALERT_RULE_NAMESPACE}.to` as const; -const ALERT_RULE_TYPE = `${ALERT_RULE_NAMESPACE}.type` as const; -const ALERT_RULE_UPDATED_AT = `${ALERT_RULE_NAMESPACE}.updated_at` as const; -const ALERT_RULE_UPDATED_BY = `${ALERT_RULE_NAMESPACE}.updated_by` as const; -const ALERT_RULE_VERSION = `${ALERT_RULE_NAMESPACE}.version` as const; // Fields pertaining to the threat tactic associated with the rule const ALERT_THREAT_FRAMEWORK = `${ALERT_RULE_THREAT_NAMESPACE}.framework` as const; @@ -186,36 +190,8 @@ export { ALERT_BUILDING_BLOCK_TYPE, ALERT_EVALUATION_THRESHOLD, ALERT_EVALUATION_VALUE, - ALERT_INSTANCE_ID, - ALERT_RISK_SCORE, - ALERT_WORKFLOW_REASON, - ALERT_WORKFLOW_USER, - ALERT_CASE_IDS, - ALERT_RULE_AUTHOR, - ALERT_RULE_CREATED_AT, - ALERT_RULE_CREATED_BY, - ALERT_RULE_DESCRIPTION, - ALERT_RULE_ENABLED, ALERT_RULE_EXCEPTIONS_LIST, - ALERT_RULE_FROM, - ALERT_RULE_INTERVAL, - ALERT_RULE_LICENSE, ALERT_RULE_NAMESPACE_FIELD, - ALERT_RULE_NOTE, - ALERT_RULE_REFERENCES, - ALERT_RULE_RULE_ID, - ALERT_RULE_RULE_NAME_OVERRIDE, - ALERT_RULE_TO, - ALERT_RULE_TYPE, - ALERT_RULE_UPDATED_AT, - ALERT_RULE_UPDATED_BY, - ALERT_RULE_VERSION, - ALERT_SEVERITY, - ALERT_SYSTEM_STATUS, - ECS_VERSION, - EVENT_ACTION, - EVENT_KIND, - EVENT_MODULE, ALERT_THREAT_FRAMEWORK, ALERT_THREAT_TACTIC_ID, ALERT_THREAT_TACTIC_NAME, @@ -226,14 +202,7 @@ export { ALERT_THREAT_TECHNIQUE_SUBTECHNIQUE_ID, ALERT_THREAT_TECHNIQUE_SUBTECHNIQUE_NAME, ALERT_THREAT_TECHNIQUE_SUBTECHNIQUE_REFERENCE, - ALERT_SUPPRESSION_TERMS, - ALERT_SUPPRESSION_FIELD, - ALERT_SUPPRESSION_VALUE, - ALERT_SUPPRESSION_START, - ALERT_SUPPRESSION_END, - ALERT_SUPPRESSION_DOCS_COUNT, - TAGS, - TIMESTAMP, + EVENT_MODULE, }; export type TechnicalRuleDataFieldName = ValuesType; diff --git a/packages/kbn-rule-data-utils/tsconfig.json b/packages/kbn-rule-data-utils/tsconfig.json index 5c94013fc2eaf0..77352c4f44209c 100644 --- a/packages/kbn-rule-data-utils/tsconfig.json +++ b/packages/kbn-rule-data-utils/tsconfig.json @@ -11,7 +11,7 @@ "**/*.ts" ], "kbn_references": [ - "@kbn/es-query" + "@kbn/es-query", ], "exclude": [ "target/**/*", diff --git a/tsconfig.base.json b/tsconfig.base.json index cff7037533bfef..6a11e26dda7ac9 100644 --- a/tsconfig.base.json +++ b/tsconfig.base.json @@ -32,6 +32,8 @@ "@kbn/alerting-plugin/*": ["x-pack/plugins/alerting/*"], "@kbn/alerts": ["packages/kbn-alerts"], "@kbn/alerts/*": ["packages/kbn-alerts/*"], + "@kbn/alerts-as-data-utils": ["packages/kbn-alerts-as-data-utils"], + "@kbn/alerts-as-data-utils/*": ["packages/kbn-alerts-as-data-utils/*"], "@kbn/alerts-restricted-fixtures-plugin": ["x-pack/test/alerting_api_integration/common/plugins/alerts_restricted"], "@kbn/alerts-restricted-fixtures-plugin/*": ["x-pack/test/alerting_api_integration/common/plugins/alerts_restricted/*"], "@kbn/alerts-ui-shared": ["packages/kbn-alerts-ui-shared"], diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts index b4cd25a4f41260..4fc36193a15d9b 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/component_template_from_field_map.ts @@ -6,8 +6,8 @@ */ import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; +import { type FieldMap } from '@kbn/alerts-as-data-utils'; import { mappingFromFieldMap } from './mapping_from_field_map'; -import { FieldMap } from './types'; export interface GetComponentTemplateFromFieldMapOpts { name: string; diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts index 2f2cac2367e8bd..f5eeeb8ba6c35c 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.test.ts @@ -4,9 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ +import { alertFieldMap, legacyAlertFieldMap, type FieldMap } from '@kbn/alerts-as-data-utils'; import { mappingFromFieldMap } from './mapping_from_field_map'; -import { FieldMap } from './types'; -import { alertFieldMap } from './alert_field_map'; describe('mappingFromFieldMap', () => { const fieldMap: FieldMap = { @@ -118,6 +117,15 @@ describe('mappingFromFieldMap', () => { date_field: { type: 'date', }, + multifield_field: { + fields: { + text: { + type: 'match_only_text', + }, + }, + ignore_above: 1024, + type: 'keyword', + }, geopoint_field: { type: 'geo_point', }, @@ -131,15 +139,6 @@ describe('mappingFromFieldMap', () => { long_field: { type: 'long', }, - multifield_field: { - fields: { - text: { - type: 'match_only_text', - }, - }, - ignore_above: 1024, - type: 'keyword', - }, nested_array_field: { properties: { field1: { @@ -184,6 +183,9 @@ describe('mappingFromFieldMap', () => { expect(mappingFromFieldMap(alertFieldMap)).toEqual({ dynamic: 'strict', properties: { + '@timestamp': { + type: 'date', + }, kibana: { properties: { alert: { @@ -191,6 +193,9 @@ describe('mappingFromFieldMap', () => { action_group: { type: 'keyword', }, + case_ids: { + type: 'keyword', + }, duration: { properties: { us: { @@ -204,8 +209,18 @@ describe('mappingFromFieldMap', () => { flapping: { type: 'boolean', }, - id: { - type: 'keyword', + flapping_history: { + type: 'boolean', + }, + instance: { + properties: { + id: { + type: 'keyword', + }, + }, + }, + last_detected: { + type: 'date', }, reason: { type: 'keyword', @@ -229,8 +244,8 @@ describe('mappingFromFieldMap', () => { type: 'keyword', }, parameters: { - type: 'object', - enabled: false, + type: 'flattened', + ignore_above: 4096, }, producer: { type: 'keyword', @@ -274,6 +289,58 @@ describe('mappingFromFieldMap', () => { }, }, }); + expect(mappingFromFieldMap(legacyAlertFieldMap)).toEqual({ + dynamic: 'strict', + properties: { + kibana: { + properties: { + alert: { + properties: { + risk_score: { type: 'float' }, + rule: { + properties: { + author: { type: 'keyword' }, + created_at: { type: 'date' }, + created_by: { type: 'keyword' }, + description: { type: 'keyword' }, + enabled: { type: 'keyword' }, + from: { type: 'keyword' }, + interval: { type: 'keyword' }, + license: { type: 'keyword' }, + note: { type: 'keyword' }, + references: { type: 'keyword' }, + rule_id: { type: 'keyword' }, + rule_name_override: { type: 'keyword' }, + to: { type: 'keyword' }, + type: { type: 'keyword' }, + updated_at: { type: 'date' }, + updated_by: { type: 'keyword' }, + version: { type: 'keyword' }, + }, + }, + severity: { type: 'keyword' }, + suppression: { + properties: { + docs_count: { type: 'long' }, + end: { type: 'date' }, + terms: { + properties: { field: { type: 'keyword' }, value: { type: 'keyword' } }, + }, + start: { type: 'date' }, + }, + }, + system_status: { type: 'keyword' }, + workflow_reason: { type: 'keyword' }, + workflow_user: { type: 'keyword' }, + }, + }, + }, + }, + ecs: { properties: { version: { type: 'keyword' } } }, + event: { properties: { action: { type: 'keyword' }, kind: { type: 'keyword' } } }, + tags: { type: 'keyword' }, + }, + }); }); it('uses dynamic setting if specified', () => { diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts index 5a1de7a995b366..9d1db8e577aa53 100644 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts +++ b/x-pack/plugins/alerting/common/alert_schema/field_maps/mapping_from_field_map.ts @@ -7,7 +7,7 @@ import type { MappingTypeMapping } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; import { set } from '@kbn/safer-lodash-set'; -import { FieldMap, MultiField } from './types'; +import type { FieldMap, MultiField } from '@kbn/alerts-as-data-utils'; export function mappingFromFieldMap( fieldMap: FieldMap, @@ -29,7 +29,6 @@ export function mappingFromFieldMap( fields.forEach((field) => { // eslint-disable-next-line @typescript-eslint/naming-convention const { name, required, array, multi_fields, ...rest } = field; - const mapped = multi_fields ? { ...rest, diff --git a/x-pack/plugins/alerting/common/alert_schema/field_maps/types.ts b/x-pack/plugins/alerting/common/alert_schema/field_maps/types.ts deleted file mode 100644 index b687cbfb0cf7de..00000000000000 --- a/x-pack/plugins/alerting/common/alert_schema/field_maps/types.ts +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -export interface MultiField { - flat_name?: string; - name: string; - type: string; -} - -export interface FieldMap { - [key: string]: { - type: string; - required: boolean; - array?: boolean; - doc_values?: boolean; - enabled?: boolean; - format?: string; - ignore_above?: number; - index?: boolean; - multi_fields?: MultiField[]; - path?: string; - scaling_factor?: number; - dynamic?: boolean | string; - }; -} diff --git a/x-pack/plugins/alerting/common/alert_schema/index.ts b/x-pack/plugins/alerting/common/alert_schema/index.ts index acca43450fe347..cccb492b10e175 100644 --- a/x-pack/plugins/alerting/common/alert_schema/index.ts +++ b/x-pack/plugins/alerting/common/alert_schema/index.ts @@ -5,5 +5,5 @@ * 2.0. */ -export { alertFieldMap } from './field_maps/alert_field_map'; +export { mappingFromFieldMap } from './field_maps/mapping_from_field_map'; export { getComponentTemplateFromFieldMap } from './field_maps/component_template_from_field_map'; diff --git a/x-pack/plugins/alerting/common/index.ts b/x-pack/plugins/alerting/common/index.ts index fafadcdb99ee43..64e5a78d21e8d2 100644 --- a/x-pack/plugins/alerting/common/index.ts +++ b/x-pack/plugins/alerting/common/index.ts @@ -24,6 +24,8 @@ export * from './parse_duration'; export * from './execution_log_types'; export * from './rule_snooze_type'; +export { mappingFromFieldMap, getComponentTemplateFromFieldMap } from './alert_schema'; + export interface AlertingFrameworkHealth { isSufficientlySecure: boolean; hasPermanentEncryptionKey: boolean; diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts index 82716f935c6b3a..ca64faa7c51ea7 100644 --- a/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.test.ts @@ -6,9 +6,11 @@ */ import { elasticsearchServiceMock, loggingSystemMock } from '@kbn/core/server/mocks'; +import { elasticsearchClientMock } from '@kbn/core-elasticsearch-client-server-mocks'; import { errors as EsErrors } from '@elastic/elasticsearch'; import { ReplaySubject, Subject } from 'rxjs'; import { AlertsService } from './alerts_service'; +import { IRuleTypeAlerts } from '../types'; let logger: ReturnType; const clusterClient = elasticsearchServiceMock.createClusterClient().asInternalUser; @@ -75,40 +77,52 @@ const IlmPutBody = { name: '.alerts-ilm-policy', }; -const getIndexTemplatePutBody = (context?: string) => ({ - name: `.alerts-${context ? context : 'test'}-default-template`, - body: { - index_patterns: [`.alerts-${context ? context : 'test'}-default-*`], - composed_of: [ - 'alerts-common-component-template', - `alerts-${context ? context : 'test'}-component-template`, - ], - template: { - settings: { - auto_expand_replicas: '0-1', - hidden: true, - 'index.lifecycle': { - name: '.alerts-ilm-policy', - rollover_alias: `.alerts-${context ? context : 'test'}-default`, +interface GetIndexTemplatePutBodyOpts { + context?: string; + useLegacyAlerts?: boolean; + useEcs?: boolean; +} +const getIndexTemplatePutBody = (opts?: GetIndexTemplatePutBodyOpts) => { + const context = opts ? opts.context : undefined; + const useLegacyAlerts = opts ? opts.useLegacyAlerts : undefined; + const useEcs = opts ? opts.useEcs : undefined; + return { + name: `.alerts-${context ? context : 'test'}-default-template`, + body: { + index_patterns: [`.alerts-${context ? context : 'test'}-default-*`], + composed_of: [ + `.alerts-${context ? context : 'test'}-mappings`, + ...(useLegacyAlerts ? ['.alerts-legacy-alert-mappings'] : []), + ...(useEcs ? ['.alerts-ecs-mappings'] : []), + '.alerts-framework-mappings', + ], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-${context ? context : 'test'}-default`, + }, + 'index.mapping.total_fields.limit': 2500, + }, + mappings: { + dynamic: false, }, - 'index.mapping.total_fields.limit': 2500, }, - mappings: { - dynamic: false, + _meta: { + managed: true, }, }, - _meta: { - managed: true, - }, - }, -}); + }; +}; -const TestRegistrationContext = { +const TestRegistrationContext: IRuleTypeAlerts = { context: 'test', fieldMap: { field: { type: 'keyword', required: false } }, }; -const AnotherRegistrationContext = { +const AnotherRegistrationContext: IRuleTypeAlerts = { context: 'another', fieldMap: { field: { type: 'keyword', required: false } }, }; @@ -145,10 +159,14 @@ describe('Alerts Service', () => { expect(alertsService.isInitialized()).toEqual(true); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; - expect(componentTemplate1.name).toEqual('alerts-common-component-template'); + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); }); test('should log error and set initialized to false if adding ILM policy throws error', async () => { @@ -185,13 +203,105 @@ describe('Alerts Service', () => { expect(alertsService.isInitialized()).toEqual(false); expect(logger.error).toHaveBeenCalledWith( - `Error installing component template alerts-common-component-template - fail` + `Error installing component template .alerts-framework-mappings - fail` ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); }); + test('should update index template field limit and retry initialization if creating/updating common component template fails with field limit error', async () => { + clusterClient.cluster.putComponentTemplate.mockRejectedValueOnce( + new EsErrors.ResponseError( + elasticsearchClientMock.createApiResponse({ + statusCode: 400, + body: { + error: { + root_cause: [ + { + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + }, + ], + type: 'illegal_argument_exception', + reason: + 'updating component template [.alerts-ecs-mappings] results in invalid composable template [.alerts-security.alerts-default-index-template] after templates are merged', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'composable template [.alerts-security.alerts-default-index-template] template after composition with component templates [.alerts-ecs-mappings, .alerts-security.alerts-mappings, .alerts-technical-mappings] is invalid', + caused_by: { + type: 'illegal_argument_exception', + reason: + 'invalid composite mappings for [.alerts-security.alerts-default-index-template]', + caused_by: { + type: 'illegal_argument_exception', + reason: 'Limit of total fields [1900] has been exceeded', + }, + }, + }, + }, + }, + }) + ) + ); + const existingIndexTemplate = { + name: 'test-template', + index_template: { + index_patterns: ['test*'], + composed_of: ['.alerts-framework-mappings'], + template: { + settings: { + auto_expand_replicas: '0-1', + hidden: true, + 'index.lifecycle': { + name: '.alerts-ilm-policy', + rollover_alias: `.alerts-empty-default`, + }, + 'index.mapping.total_fields.limit': 1800, + }, + mappings: { + dynamic: false, + }, + }, + }, + }; + clusterClient.indices.getIndexTemplate.mockResolvedValueOnce({ + index_templates: [existingIndexTemplate], + }); + const alertsService = new AlertsService({ + logger, + elasticsearchClientPromise: Promise.resolve(clusterClient), + pluginStop$, + }); + + alertsService.initialize(); + await new Promise((r) => setTimeout(r, 50)); + + expect(alertsService.isInitialized()).toEqual(true); + expect(clusterClient.indices.getIndexTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ + name: existingIndexTemplate.name, + body: { + ...existingIndexTemplate.index_template, + template: { + ...existingIndexTemplate.index_template.template, + settings: { + ...existingIndexTemplate.index_template.template?.settings, + 'index.mapping.total_fields.limit': 2500, + }, + }, + }, + }); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); + // 3x for framework, legacy-alert and ecs mappings, then 1 extra time to update component template + // after updating index template field limit + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + }); + test('should install resources for contexts awaiting initialization when common resources are initialized', async () => { const alertsService = new AlertsService({ logger, @@ -214,20 +324,24 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - // 1x for common component template, 2x for context specific - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + // 1x for framework component template, 1x for legacy alert, 1x for ecs, 2x for context specific + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(5); const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; - expect(componentTemplate1.name).toEqual('alerts-common-component-template'); + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; - expect(componentTemplate2.name).toEqual('alerts-another-component-template'); + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; - expect(componentTemplate3.name).toEqual('alerts-test-component-template'); + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); + const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; + expect(componentTemplate4.name).toEqual('.alerts-another-mappings'); + const componentTemplate5 = clusterClient.cluster.putComponentTemplate.mock.calls[4][0]; + expect(componentTemplate5.name).toEqual('.alerts-test-mappings'); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledTimes(2); expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( 1, - getIndexTemplatePutBody('another') + getIndexTemplatePutBody({ context: 'another' }) ); expect(clusterClient.indices.putIndexTemplate).toHaveBeenNthCalledWith( 2, @@ -291,11 +405,15 @@ describe('Alerts Service', () => { expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; - expect(componentTemplate1.name).toEqual('alerts-common-component-template'); + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; - expect(componentTemplate2.name).toEqual('alerts-test-component-template'); + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); + const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; + expect(componentTemplate4.name).toEqual('.alerts-test-mappings'); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( getIndexTemplatePutBody() @@ -318,7 +436,87 @@ describe('Alerts Service', () => { }); }); - test('should not install component template for context fieldMap is empty', async () => { + test('should correctly install resources for context when useLegacyAlerts is true', async () => { + alertsService.register({ ...TestRegistrationContext, useLegacyAlerts: true }); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); + const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; + expect(componentTemplate4.name).toEqual('.alerts-test-mappings'); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( + getIndexTemplatePutBody({ useLegacyAlerts: true }) + ); + expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ + index: '.alerts-test-default-*', + }); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.alerts-test-default-000001', + body: { + aliases: { + '.alerts-test-default': { + is_write_index: true, + }, + }, + }, + }); + }); + + test('should correctly install resources for context when useEcs is true', async () => { + alertsService.register({ ...TestRegistrationContext, useEcs: true }); + await new Promise((r) => setTimeout(r, 50)); + expect(await alertsService.isContextInitialized(TestRegistrationContext.context)).toEqual( + true + ); + + expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); + + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); + const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); + const componentTemplate4 = clusterClient.cluster.putComponentTemplate.mock.calls[3][0]; + expect(componentTemplate4.name).toEqual('.alerts-test-mappings'); + + expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith( + getIndexTemplatePutBody({ useEcs: true }) + ); + expect(clusterClient.indices.getAlias).toHaveBeenCalledWith({ + index: '.alerts-test-default-*', + }); + expect(clusterClient.indices.putSettings).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.simulateIndexTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.putMapping).toHaveBeenCalledTimes(2); + expect(clusterClient.indices.create).toHaveBeenCalledWith({ + index: '.alerts-test-default-000001', + body: { + aliases: { + '.alerts-test-default': { + is_write_index: true, + }, + }, + }, + }); + }); + + test('should not install component template for context if fieldMap is empty', async () => { alertsService.register({ context: 'empty', fieldMap: {}, @@ -328,15 +526,19 @@ describe('Alerts Service', () => { expect(clusterClient.ilm.putLifecycle).toHaveBeenCalledWith(IlmPutBody); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); const componentTemplate1 = clusterClient.cluster.putComponentTemplate.mock.calls[0][0]; - expect(componentTemplate1.name).toEqual('alerts-common-component-template'); + expect(componentTemplate1.name).toEqual('.alerts-framework-mappings'); + const componentTemplate2 = clusterClient.cluster.putComponentTemplate.mock.calls[1][0]; + expect(componentTemplate2.name).toEqual('.alerts-legacy-alert-mappings'); + const componentTemplate3 = clusterClient.cluster.putComponentTemplate.mock.calls[2][0]; + expect(componentTemplate3.name).toEqual('.alerts-ecs-mappings'); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalledWith({ name: `.alerts-empty-default-template`, body: { index_patterns: [`.alerts-empty-default-*`], - composed_of: ['alerts-common-component-template'], + composed_of: ['.alerts-framework-mappings'], template: { settings: { auto_expand_replicas: '0-1', @@ -410,7 +612,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); // putIndexTemplate is skipped but other operations are called as expected expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); @@ -443,7 +645,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).not.toHaveBeenCalled(); expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); @@ -467,7 +669,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).not.toHaveBeenCalled(); @@ -491,7 +693,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); @@ -512,7 +714,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putSettings).not.toHaveBeenCalled(); @@ -535,7 +737,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -559,7 +761,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -581,7 +783,7 @@ describe('Alerts Service', () => { expect(logger.error).toHaveBeenCalledWith(`Failed to PUT mapping for alias alias_1: fail`); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -601,7 +803,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -640,7 +842,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -673,7 +875,7 @@ describe('Alerts Service', () => { ); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -695,7 +897,7 @@ describe('Alerts Service', () => { expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -730,7 +932,7 @@ describe('Alerts Service', () => { expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -766,7 +968,7 @@ describe('Alerts Service', () => { expect(logger.error).toHaveBeenCalledWith(`Error creating concrete write index - fail`); expect(clusterClient.ilm.putLifecycle).toHaveBeenCalled(); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(4); expect(clusterClient.indices.simulateTemplate).toHaveBeenCalled(); expect(clusterClient.indices.putIndexTemplate).toHaveBeenCalled(); expect(clusterClient.indices.getAlias).toHaveBeenCalled(); @@ -810,7 +1012,7 @@ describe('Alerts Service', () => { alertsService.initialize(); await new Promise((r) => setTimeout(r, 150)); expect(alertsService.isInitialized()).toEqual(true); - expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(3); + expect(clusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(5); }); test('should retry updating index template for transient ES errors', async () => { diff --git a/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts b/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts index 08643caf862f5a..22cb9f4df18840 100644 --- a/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts +++ b/x-pack/plugins/alerting/server/alerts_service/alerts_service.ts @@ -13,8 +13,14 @@ import { import { get, isEmpty, isEqual } from 'lodash'; import { Logger, ElasticsearchClient } from '@kbn/core/server'; import { firstValueFrom, Observable } from 'rxjs'; -import { FieldMap } from '../../common/alert_schema/field_maps/types'; -import { alertFieldMap } from '../../common/alert_schema'; +import { + alertFieldMap, + ecsFieldMap, + legacyAlertFieldMap, + type FieldMap, +} from '@kbn/alerts-as-data-utils'; +import { IndicesGetIndexTemplateIndexTemplateItem } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; +import { asyncForEach } from '@kbn/std'; import { DEFAULT_ALERTS_ILM_POLICY_NAME, DEFAULT_ALERTS_ILM_POLICY, @@ -34,7 +40,9 @@ import { const TOTAL_FIELDS_LIMIT = 2500; const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes - +const LEGACY_ALERT_CONTEXT = 'legacy-alert'; +export const ECS_CONTEXT = `ecs`; +export const ECS_COMPONENT_TEMPLATE_NAME = getComponentTemplateName(ECS_CONTEXT); interface AlertsServiceParams { logger: Logger; pluginStop$: Observable; @@ -107,6 +115,16 @@ export class AlertsService implements IAlertsService { const initFns = [ () => this.createOrUpdateIlmPolicy(esClient), () => this.createOrUpdateComponentTemplate(esClient, getComponentTemplate(alertFieldMap)), + () => + this.createOrUpdateComponentTemplate( + esClient, + getComponentTemplate(legacyAlertFieldMap, LEGACY_ALERT_CONTEXT) + ), + () => + this.createOrUpdateComponentTemplate( + esClient, + getComponentTemplate(ecsFieldMap, ECS_CONTEXT) + ), ]; for (const fn of initFns) { @@ -127,7 +145,8 @@ export class AlertsService implements IAlertsService { }); } - public register({ context, fieldMap }: IRuleTypeAlerts, timeoutMs?: number) { + public register(opts: IRuleTypeAlerts, timeoutMs?: number) { + const { context, fieldMap } = opts; // check whether this context has been registered before if (this.registeredContexts.has(context)) { const registeredFieldMap = this.registeredContexts.get(context); @@ -140,37 +159,54 @@ export class AlertsService implements IAlertsService { this.options.logger.info(`Registering resources for context "${context}".`); this.registeredContexts.set(context, fieldMap); - this.resourceInitializationHelper.add({ context, fieldMap }, timeoutMs); + this.resourceInitializationHelper.add(opts, timeoutMs); } - private async initializeContext({ context, fieldMap }: IRuleTypeAlerts, timeoutMs?: number) { + private async initializeContext( + { context, fieldMap, useEcs, useLegacyAlerts }: IRuleTypeAlerts, + timeoutMs?: number + ) { const esClient = await this.options.elasticsearchClientPromise; const indexTemplateAndPattern = getIndexTemplateAndPattern(context); - // Context specific initialization installs component template, index template and write index - // If fieldMap is empty, don't create context specific component template - const initFns = isEmpty(fieldMap) - ? [ - async () => - await this.createOrUpdateIndexTemplate(esClient, indexTemplateAndPattern, [ - getComponentTemplateName(), - ]), - async () => await this.createConcreteWriteIndex(esClient, indexTemplateAndPattern), - ] - : [ - async () => - await this.createOrUpdateComponentTemplate( - esClient, - getComponentTemplate(fieldMap, context) - ), - async () => - await this.createOrUpdateIndexTemplate(esClient, indexTemplateAndPattern, [ - getComponentTemplateName(), - getComponentTemplateName(context), - ]), - async () => await this.createConcreteWriteIndex(esClient, indexTemplateAndPattern), - ]; + let initFns: Array<() => Promise> = []; + + // List of component templates to reference + const componentTemplateRefs: string[] = []; + + // If fieldMap is not empty, create a context specific component template + if (!isEmpty(fieldMap)) { + const componentTemplate = getComponentTemplate(fieldMap, context); + initFns.push( + async () => await this.createOrUpdateComponentTemplate(esClient, componentTemplate) + ); + componentTemplateRefs.push(componentTemplate.name); + } + + // If useLegacy is set to true, add the legacy alert component template to the references + if (useLegacyAlerts) { + componentTemplateRefs.push(getComponentTemplateName(LEGACY_ALERT_CONTEXT)); + } + + // If useEcs is set to true, add the ECS component template to the references + if (useEcs) { + componentTemplateRefs.push(getComponentTemplateName(ECS_CONTEXT)); + } + + // Add framework component template to the references + componentTemplateRefs.push(getComponentTemplateName()); + + // Context specific initialization installs index template and write index + initFns = initFns.concat([ + async () => + await this.createOrUpdateIndexTemplate( + esClient, + indexTemplateAndPattern, + componentTemplateRefs + ), + async () => await this.createConcreteWriteIndex(esClient, indexTemplateAndPattern), + ]); for (const fn of initFns) { await this.installWithTimeout(async () => await fn(), timeoutMs); @@ -200,6 +236,62 @@ export class AlertsService implements IAlertsService { } } + private async getIndexTemplatesUsingComponentTemplate( + esClient: ElasticsearchClient, + componentTemplateName: string + ) { + // Get all index templates and filter down to just the ones referencing this component template + const { index_templates: indexTemplates } = await esClient.indices.getIndexTemplate(); + const indexTemplatesUsingComponentTemplate = (indexTemplates ?? []).filter( + (indexTemplate: IndicesGetIndexTemplateIndexTemplateItem) => + indexTemplate.index_template.composed_of.includes(componentTemplateName) + ); + await asyncForEach( + indexTemplatesUsingComponentTemplate, + async (template: IndicesGetIndexTemplateIndexTemplateItem) => { + await esClient.indices.putIndexTemplate({ + name: template.name, + body: { + ...template.index_template, + template: { + ...template.index_template.template, + settings: { + ...template.index_template.template?.settings, + 'index.mapping.total_fields.limit': TOTAL_FIELDS_LIMIT, + }, + }, + }, + }); + } + ); + } + + private async createOrUpdateComponentTemplateHelper( + esClient: ElasticsearchClient, + template: ClusterPutComponentTemplateRequest + ) { + try { + await esClient.cluster.putComponentTemplate(template); + } catch (error) { + const reason = error?.meta?.body?.error?.caused_by?.caused_by?.caused_by?.reason; + if (reason && reason.match(/Limit of total fields \[\d+\] has been exceeded/) != null) { + // This error message occurs when there is an index template using this component template + // that contains a field limit setting that using this component template exceeds + // Specifically, this can happen for the ECS component template when we add new fields + // to adhere to the ECS spec. Individual index templates specify field limits so if the + // number of new ECS fields pushes the composed mapping above the limit, this error will + // occur. We have to update the field limit inside the index template now otherwise we + // can never update the component template + await this.getIndexTemplatesUsingComponentTemplate(esClient, template.name); + + // Try to update the component template again + await esClient.cluster.putComponentTemplate(template); + } else { + throw error; + } + } + } + private async createOrUpdateComponentTemplate( esClient: ElasticsearchClient, template: ClusterPutComponentTemplateRequest @@ -207,9 +299,12 @@ export class AlertsService implements IAlertsService { this.options.logger.info(`Installing component template ${template.name}`); try { - await retryTransientEsErrors(() => esClient.cluster.putComponentTemplate(template), { - logger: this.options.logger, - }); + await retryTransientEsErrors( + () => this.createOrUpdateComponentTemplateHelper(esClient, template), + { + logger: this.options.logger, + } + ); } catch (err) { this.options.logger.error( `Error installing component template ${template.name} - ${err.message}` diff --git a/x-pack/plugins/rule_registry/common/field_map/types.ts b/x-pack/plugins/alerting/server/alerts_service/index.ts similarity index 52% rename from x-pack/plugins/rule_registry/common/field_map/types.ts rename to x-pack/plugins/alerting/server/alerts_service/index.ts index 52ee246375ad05..49247f3baa243a 100644 --- a/x-pack/plugins/rule_registry/common/field_map/types.ts +++ b/x-pack/plugins/alerting/server/alerts_service/index.ts @@ -5,13 +5,9 @@ * 2.0. */ -export interface FieldMap { - [key: string]: { - type: string; - required?: boolean; - array?: boolean; - path?: string; - scaling_factor?: number; - dynamic?: 'strict' | boolean; - }; -} +export { + DEFAULT_ALERTS_ILM_POLICY, + DEFAULT_ALERTS_ILM_POLICY_NAME, +} from './default_lifecycle_policy'; +export { ECS_COMPONENT_TEMPLATE_NAME, ECS_CONTEXT } from './alerts_service'; +export { getComponentTemplate } from './types'; diff --git a/x-pack/plugins/alerting/server/alerts_service/types.ts b/x-pack/plugins/alerting/server/alerts_service/types.ts index db47a9a8e00150..aeb73cab6ffd21 100644 --- a/x-pack/plugins/alerting/server/alerts_service/types.ts +++ b/x-pack/plugins/alerting/server/alerts_service/types.ts @@ -6,11 +6,11 @@ */ import { ClusterPutComponentTemplateRequest } from '@elastic/elasticsearch/lib/api/types'; -import { getComponentTemplateFromFieldMap } from '../../common/alert_schema'; -import { FieldMap } from '../../common/alert_schema/field_maps/types'; +import type { FieldMap } from '@kbn/alerts-as-data-utils'; +import { getComponentTemplateFromFieldMap } from '../../common'; export const getComponentTemplateName = (context?: string) => - `alerts-${context ? context : 'common'}-component-template`; + `.alerts-${context || 'framework'}-mappings`; export interface IIndexPatternString { template: string; @@ -40,5 +40,5 @@ export const getComponentTemplate = ( name: getComponentTemplateName(context), fieldMap, // set field limit slightly higher than actual number of fields - fieldLimit: 100, // Math.round(Object.keys(fieldMap).length * 1.5), + fieldLimit: Math.ceil(Object.keys(fieldMap).length / 1000) * 1000 + 500, }); diff --git a/x-pack/plugins/alerting/server/index.ts b/x-pack/plugins/alerting/server/index.ts index a13b06596f557b..6b5ad3012d8a91 100644 --- a/x-pack/plugins/alerting/server/index.ts +++ b/x-pack/plugins/alerting/server/index.ts @@ -56,7 +56,10 @@ export { export { DEFAULT_ALERTS_ILM_POLICY, DEFAULT_ALERTS_ILM_POLICY_NAME, -} from './alerts_service/default_lifecycle_policy'; + ECS_COMPONENT_TEMPLATE_NAME, + ECS_CONTEXT, + getComponentTemplate, +} from './alerts_service'; export const plugin = (initContext: PluginInitializerContext) => new AlertingPlugin(initContext); diff --git a/x-pack/plugins/alerting/server/types.ts b/x-pack/plugins/alerting/server/types.ts index e319f315440af5..4997d9563d59e0 100644 --- a/x-pack/plugins/alerting/server/types.ts +++ b/x-pack/plugins/alerting/server/types.ts @@ -22,6 +22,7 @@ import { } from '@kbn/core/server'; import type { PublicMethodsOf } from '@kbn/utility-types'; import { SharePluginStart } from '@kbn/share-plugin/server'; +import { type FieldMap } from '@kbn/alerts-as-data-utils'; import { RuleTypeRegistry as OrigruleTypeRegistry } from './rule_type_registry'; import { PluginSetupContract, PluginStartContract } from './plugin'; import { RulesClient } from './rules_client'; @@ -51,7 +52,6 @@ import { SanitizedRule, } from '../common'; import { PublicAlertFactory } from './alert/create_alert_factory'; -import { FieldMap } from '../common/alert_schema/field_maps/types'; import { RulesSettingsFlappingProperties } from '../common/rules_settings'; export type WithoutQueryAndParams = Pick>; export type SpaceIdToNamespaceFunction = (spaceId?: string) => string | undefined; @@ -172,6 +172,8 @@ export interface IRuleTypeAlerts { context: string; namespace?: string; fieldMap: FieldMap; + useEcs?: boolean; + useLegacyAlerts?: boolean; } export interface RuleType< diff --git a/x-pack/plugins/alerting/tsconfig.json b/x-pack/plugins/alerting/tsconfig.json index 1f7017de59a2e1..6fed19209ad122 100644 --- a/x-pack/plugins/alerting/tsconfig.json +++ b/x-pack/plugins/alerting/tsconfig.json @@ -39,6 +39,8 @@ "@kbn/data-views-plugin", "@kbn/share-plugin", "@kbn/safer-lodash-set", + "@kbn/alerts-as-data-utils", + "@kbn/core-elasticsearch-client-server-mocks", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index 86907fdd570bee..5b98c8e437782a 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -15,10 +15,10 @@ import { PluginInitializerContext, } from '@kbn/core/server'; import { isEmpty, mapValues } from 'lodash'; -import { mappingFromFieldMap } from '@kbn/rule-registry-plugin/common/mapping_from_field_map'; import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; import { Dataset } from '@kbn/rule-registry-plugin/server'; import { UI_SETTINGS } from '@kbn/data-plugin/common'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { APMConfig, APM_SERVER_FEATURE_ID } from '.'; import { APM_FEATURE, registerFeaturesUsage } from './feature'; import { registerApmRuleTypes } from './routes/alerts/register_apm_rule_types'; @@ -130,25 +130,32 @@ export class APMPlugin ...experimentalRuleFieldMap, [SERVICE_NAME]: { type: 'keyword', + required: false, }, [SERVICE_ENVIRONMENT]: { type: 'keyword', + required: false, }, [TRANSACTION_TYPE]: { type: 'keyword', + required: false, }, [PROCESSOR_EVENT]: { type: 'keyword', + required: false, }, [AGENT_NAME]: { type: 'keyword', + required: false, }, [SERVICE_LANGUAGE_NAME]: { type: 'keyword', + required: false, }, labels: { type: 'object', dynamic: true, + required: false, }, }, 'strict' diff --git a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts index 1435f4812d16ca..3a81f957e93141 100644 --- a/x-pack/plugins/infra/server/services/rules/rule_data_client.ts +++ b/x-pack/plugins/infra/server/services/rules/rule_data_client.ts @@ -6,11 +6,11 @@ */ import { CoreSetup, Logger } from '@kbn/core/server'; -import { mappingFromFieldMap } from '@kbn/rule-registry-plugin/common/mapping_from_field_map'; import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; import { Dataset, RuleRegistryPluginSetupContract } from '@kbn/rule-registry-plugin/server'; -import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/rule-registry-plugin/common/assets'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; +import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/alerting-plugin/server'; import type { InfraFeatureId } from '../../../common/constants'; import { RuleRegistrationContext, RulesServiceStartDeps } from './types'; diff --git a/x-pack/plugins/observability/server/plugin.ts b/x-pack/plugins/observability/server/plugin.ts index 98b21abbaefe5c..39a69d7fa16667 100644 --- a/x-pack/plugins/observability/server/plugin.ts +++ b/x-pack/plugins/observability/server/plugin.ts @@ -18,10 +18,10 @@ import { Dataset, RuleRegistryPluginSetupContract } from '@kbn/rule-registry-plu import { PluginSetupContract as FeaturesSetup } from '@kbn/features-plugin/server'; import { createUICapabilities } from '@kbn/cases-plugin/common'; import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; -import { mappingFromFieldMap } from '@kbn/rule-registry-plugin/common/mapping_from_field_map'; -import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/rule-registry-plugin/common/assets'; +import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/alerting-plugin/server'; import type { GuidedOnboardingPluginSetup } from '@kbn/guided-onboarding-plugin/server'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { UsageCollectionSetup } from '@kbn/usage-collection-plugin/server'; import { kubernetesGuideId, diff --git a/x-pack/plugins/rule_registry/common/assets.ts b/x-pack/plugins/rule_registry/common/assets.ts index a1df09df18a8f9..1e8919a3a07e4a 100644 --- a/x-pack/plugins/rule_registry/common/assets.ts +++ b/x-pack/plugins/rule_registry/common/assets.ts @@ -5,5 +5,4 @@ * 2.0. */ -export const TECHNICAL_COMPONENT_TEMPLATE_NAME = `technical-mappings`; -export const ECS_COMPONENT_TEMPLATE_NAME = `ecs-mappings`; +export const TECHNICAL_COMPONENT_TEMPLATE_NAME = `.alerts-technical-mappings`; diff --git a/x-pack/plugins/rule_registry/common/assets/component_templates/ecs_component_template.ts b/x-pack/plugins/rule_registry/common/assets/component_templates/ecs_component_template.ts index 8e956ba0004a24..8f30e07a0d9dcf 100644 --- a/x-pack/plugins/rule_registry/common/assets/component_templates/ecs_component_template.ts +++ b/x-pack/plugins/rule_registry/common/assets/component_templates/ecs_component_template.ts @@ -5,7 +5,7 @@ * 2.0. */ import { merge } from 'lodash'; -import { mappingFromFieldMap } from '../../mapping_from_field_map'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { ClusterPutComponentTemplateBody } from '../../types'; import { ecsFieldMap } from '../field_maps/ecs_field_map'; import { technicalRuleFieldMap } from '../field_maps/technical_rule_field_map'; diff --git a/x-pack/plugins/rule_registry/common/assets/component_templates/technical_component_template.ts b/x-pack/plugins/rule_registry/common/assets/component_templates/technical_component_template.ts index e110be339d0a0f..1315d7f0d1b587 100644 --- a/x-pack/plugins/rule_registry/common/assets/component_templates/technical_component_template.ts +++ b/x-pack/plugins/rule_registry/common/assets/component_templates/technical_component_template.ts @@ -5,7 +5,7 @@ * 2.0. */ -import { mappingFromFieldMap } from '../../mapping_from_field_map'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { ClusterPutComponentTemplateBody } from '../../types'; import { technicalRuleFieldMap } from '../field_maps/technical_rule_field_map'; diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.test.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.test.ts index 4e2d591bf88bd0..3a6dbc4f209827 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.test.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.test.ts @@ -13,10 +13,12 @@ it('matches snapshot', () => { expect(experimentalRuleFieldMap).toMatchInlineSnapshot(` Object { "kibana.alert.evaluation.threshold": Object { + "required": false, "scaling_factor": 100, "type": "scaled_float", }, "kibana.alert.evaluation.value": Object { + "required": false, "scaling_factor": 100, "type": "scaled_float", }, diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.ts index 92f93015309c0b..3859ebe6df9b6b 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/experimental_rule_field_map.ts @@ -8,8 +8,12 @@ import * as Fields from '../../technical_rule_data_field_names'; export const experimentalRuleFieldMap = { - [Fields.ALERT_EVALUATION_THRESHOLD]: { type: 'scaled_float', scaling_factor: 100 }, - [Fields.ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100 }, + [Fields.ALERT_EVALUATION_THRESHOLD]: { + type: 'scaled_float', + scaling_factor: 100, + required: false, + }, + [Fields.ALERT_EVALUATION_VALUE]: { type: 'scaled_float', scaling_factor: 100, required: false }, } as const; export type ExperimentalRuleFieldMap = typeof experimentalRuleFieldMap; diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts index e7d39a71a1d6d1..7c2cc7c2a02af0 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.test.ts @@ -43,15 +43,27 @@ it('matches snapshot', () => { "type": "keyword", }, "kibana.alert.duration.us": Object { + "array": false, + "required": false, "type": "long", }, "kibana.alert.end": Object { + "array": false, + "required": false, "type": "date", }, "kibana.alert.flapping": Object { + "array": false, + "required": false, + "type": "boolean", + }, + "kibana.alert.flapping_history": Object { + "array": true, + "required": false, "type": "boolean", }, "kibana.alert.instance.id": Object { + "array": false, "required": true, "type": "keyword", }, @@ -81,6 +93,7 @@ it('matches snapshot', () => { "type": "keyword", }, "kibana.alert.rule.consumer": Object { + "array": false, "required": true, "type": "keyword", }, @@ -135,10 +148,13 @@ it('matches snapshot', () => { "type": "keyword", }, "kibana.alert.rule.parameters": Object { + "array": false, "ignore_above": 4096, + "required": false, "type": "flattened", }, "kibana.alert.rule.producer": Object { + "array": false, "required": true, "type": "keyword", }, @@ -158,6 +174,7 @@ it('matches snapshot', () => { "type": "keyword", }, "kibana.alert.rule.rule_type_id": Object { + "array": false, "required": true, "type": "keyword", }, @@ -197,12 +214,17 @@ it('matches snapshot', () => { "type": "keyword", }, "kibana.alert.severity": Object { + "array": false, + "required": false, "type": "keyword", }, "kibana.alert.start": Object { + "array": false, + "required": false, "type": "date", }, "kibana.alert.status": Object { + "array": false, "required": true, "type": "keyword", }, @@ -237,10 +259,13 @@ it('matches snapshot', () => { "type": "keyword", }, "kibana.alert.time_range": Object { + "array": false, "format": "epoch_millis||strict_date_optional_time", + "required": false, "type": "date_range", }, "kibana.alert.uuid": Object { + "array": false, "required": true, "type": "keyword", }, diff --git a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts index 32d1b45e44cada..ef476f468544b5 100644 --- a/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts +++ b/x-pack/plugins/rule_registry/common/assets/field_maps/technical_rule_field_map.ts @@ -5,225 +5,12 @@ * 2.0. */ +import { alertFieldMap, legacyAlertFieldMap } from '@kbn/alerts-as-data-utils'; import { pickWithPatterns } from '../../pick_with_patterns'; -import * as Fields from '../../technical_rule_data_field_names'; -import { ecsFieldMap } from './ecs_field_map'; export const technicalRuleFieldMap = { - ...pickWithPatterns( - ecsFieldMap, - Fields.TIMESTAMP, - Fields.EVENT_KIND, - Fields.EVENT_ACTION, - Fields.TAGS - ), - [Fields.ALERT_RULE_PARAMETERS]: { type: 'flattened', ignore_above: 4096 }, - [Fields.ALERT_RULE_TYPE_ID]: { type: 'keyword', required: true }, - [Fields.ALERT_RULE_CONSUMER]: { type: 'keyword', required: true }, - [Fields.ALERT_RULE_PRODUCER]: { type: 'keyword', required: true }, - [Fields.SPACE_IDS]: { type: 'keyword', array: true, required: true }, - [Fields.ALERT_UUID]: { type: 'keyword', required: true }, - [Fields.ALERT_INSTANCE_ID]: { type: 'keyword', required: true }, - [Fields.ALERT_START]: { type: 'date' }, - [Fields.ALERT_TIME_RANGE]: { - type: 'date_range', - format: 'epoch_millis||strict_date_optional_time', - }, - [Fields.ALERT_END]: { type: 'date' }, - [Fields.ALERT_DURATION]: { type: 'long' }, - [Fields.ALERT_SEVERITY]: { type: 'keyword' }, - [Fields.ALERT_STATUS]: { type: 'keyword', required: true }, - [Fields.ALERT_FLAPPING]: { type: 'boolean' }, - [Fields.VERSION]: { - type: 'version', - array: false, - required: false, - }, - [Fields.ECS_VERSION]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RISK_SCORE]: { - type: 'float', - array: false, - required: false, - }, - [Fields.ALERT_WORKFLOW_STATUS]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_WORKFLOW_USER]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_WORKFLOW_REASON]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_SYSTEM_STATUS]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_ACTION_GROUP]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_REASON]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_CASE_IDS]: { - type: 'keyword', - array: true, - required: false, - }, - [Fields.ALERT_RULE_AUTHOR]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_CATEGORY]: { - type: 'keyword', - array: false, - required: true, - }, - [Fields.ALERT_RULE_UUID]: { - type: 'keyword', - array: false, - required: true, - }, - [Fields.ALERT_RULE_CREATED_AT]: { - type: 'date', - array: false, - required: false, - }, - [Fields.ALERT_RULE_CREATED_BY]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_DESCRIPTION]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_ENABLED]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_EXECUTION_UUID]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_FROM]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_INTERVAL]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_LICENSE]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_NAME]: { - type: 'keyword', - array: false, - required: true, - }, - [Fields.ALERT_RULE_NOTE]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_REFERENCES]: { - type: 'keyword', - array: true, - required: false, - }, - [Fields.ALERT_RULE_RULE_ID]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_RULE_NAME_OVERRIDE]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_TAGS]: { - type: 'keyword', - array: true, - required: false, - }, - [Fields.ALERT_RULE_TO]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_TYPE]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_UPDATED_AT]: { - type: 'date', - array: false, - required: false, - }, - [Fields.ALERT_RULE_UPDATED_BY]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_RULE_VERSION]: { - type: 'keyword', - array: false, - required: false, - }, - [Fields.ALERT_SUPPRESSION_FIELD]: { - type: 'keyword', - array: true, - required: false, - }, - [Fields.ALERT_SUPPRESSION_VALUE]: { - type: 'keyword', - array: true, - required: false, - }, - [Fields.ALERT_SUPPRESSION_START]: { - type: 'date', - array: false, - required: false, - }, - [Fields.ALERT_SUPPRESSION_END]: { - type: 'date', - array: false, - required: false, - }, - [Fields.ALERT_SUPPRESSION_DOCS_COUNT]: { - type: 'long', - array: false, - required: false, - }, - [Fields.ALERT_LAST_DETECTED]: { - type: 'date', - array: false, - required: false, - }, + ...pickWithPatterns(alertFieldMap, '*'), + ...pickWithPatterns(legacyAlertFieldMap, '*'), } as const; export type TechnicalRuleFieldMap = typeof technicalRuleFieldMap; diff --git a/x-pack/plugins/rule_registry/common/field_map/index.ts b/x-pack/plugins/rule_registry/common/field_map/index.ts index fac8575b8af48e..e64ba5823e6739 100644 --- a/x-pack/plugins/rule_registry/common/field_map/index.ts +++ b/x-pack/plugins/rule_registry/common/field_map/index.ts @@ -7,4 +7,3 @@ export * from './merge_field_maps'; export * from './runtime_type_from_fieldmap'; -export * from './types'; diff --git a/x-pack/plugins/rule_registry/common/field_map/merge_field_maps.ts b/x-pack/plugins/rule_registry/common/field_map/merge_field_maps.ts index 124de243352ea3..701bab82855d44 100644 --- a/x-pack/plugins/rule_registry/common/field_map/merge_field_maps.ts +++ b/x-pack/plugins/rule_registry/common/field_map/merge_field_maps.ts @@ -4,7 +4,8 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import { FieldMap } from './types'; + +import type { FieldMap } from '@kbn/alerts-as-data-utils'; export function mergeFieldMaps( first: T1, diff --git a/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.test.ts b/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.test.ts index 8ee71356ef7064..0b724150f0dcc2 100644 --- a/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.test.ts +++ b/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.test.ts @@ -8,11 +8,11 @@ import { runtimeTypeFromFieldMap } from './runtime_type_from_fieldmap'; describe('runtimeTypeFromFieldMap', () => { const fieldmapRt = runtimeTypeFromFieldMap({ - keywordField: { type: 'keyword' }, - longField: { type: 'long' }, - booleanField: { type: 'boolean' }, + keywordField: { type: 'keyword', required: false }, + longField: { type: 'long', required: false }, + booleanField: { type: 'boolean', required: false }, requiredKeywordField: { type: 'keyword', required: true }, - multiKeywordField: { type: 'keyword', array: true }, + multiKeywordField: { type: 'keyword', array: true, required: false }, } as const); it('accepts both singular and array fields', () => { diff --git a/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts b/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts index feb59f88abc7b0..93e182e53af633 100644 --- a/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts +++ b/x-pack/plugins/rule_registry/common/field_map/runtime_type_from_fieldmap.ts @@ -8,7 +8,7 @@ import { Optional } from 'utility-types'; import { mapValues, pickBy } from 'lodash'; import { either } from 'fp-ts/lib/Either'; import * as t from 'io-ts'; -import { FieldMap } from './types'; +import type { FieldMap } from '@kbn/alerts-as-data-utils'; const NumberFromString = new t.Type( 'NumberFromString', diff --git a/x-pack/plugins/rule_registry/common/mapping_from_field_map.ts b/x-pack/plugins/rule_registry/common/mapping_from_field_map.ts deleted file mode 100644 index 1b66496bee19b8..00000000000000 --- a/x-pack/plugins/rule_registry/common/mapping_from_field_map.ts +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import type * as estypes from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; -import { set } from '@kbn/safer-lodash-set'; -import { FieldMap } from './field_map/types'; - -export function mappingFromFieldMap( - fieldMap: FieldMap, - dynamic: 'strict' | boolean -): estypes.MappingTypeMapping { - const mappings = { - dynamic, - properties: {}, - }; - - const fields = Object.keys(fieldMap).map((key) => { - const field = fieldMap[key]; - return { - name: key, - ...field, - }; - }); - - fields.forEach((field) => { - const { name, required, array, ...rest } = field; - - set(mappings.properties, field.name.split('.').join('.properties.'), rest); - }); - - return mappings; -} diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts index b63fb2aae83d03..083c4d08d42536 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.test.ts @@ -12,11 +12,9 @@ import { AlertConsumers } from '@kbn/rule-data-utils'; import { Dataset } from './index_options'; import { IndexInfo } from './index_info'; +import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/alerting-plugin/server'; import { elasticsearchServiceMock, ElasticsearchClientMock } from '@kbn/core/server/mocks'; -import { - ECS_COMPONENT_TEMPLATE_NAME, - TECHNICAL_COMPONENT_TEMPLATE_NAME, -} from '../../common/assets'; +import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../common/assets'; describe('resourceInstaller', () => { let pluginStop$: Subject; @@ -82,15 +80,11 @@ describe('resourceInstaller', () => { it('should install common resources', async () => { const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); const getClusterClient = jest.fn(() => Promise.resolve(mockClusterClient)); - const getResourceNameMock = jest - .fn() - .mockReturnValueOnce(TECHNICAL_COMPONENT_TEMPLATE_NAME) - .mockReturnValueOnce(ECS_COMPONENT_TEMPLATE_NAME); const installer = new ResourceInstaller({ logger: loggerMock.create(), isWriteEnabled: true, disabledRegistrationContexts: [], - getResourceName: getResourceNameMock, + getResourceName: jest.fn(), getClusterClient, areFrameworkAlertsEnabled: false, pluginStop$, @@ -102,26 +96,22 @@ describe('resourceInstaller', () => { expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenNthCalledWith( 1, - expect.objectContaining({ name: TECHNICAL_COMPONENT_TEMPLATE_NAME }) + expect.objectContaining({ name: ECS_COMPONENT_TEMPLATE_NAME }) ); expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenNthCalledWith( 2, - expect.objectContaining({ name: ECS_COMPONENT_TEMPLATE_NAME }) + expect.objectContaining({ name: TECHNICAL_COMPONENT_TEMPLATE_NAME }) ); }); - it('should install common resources when framework alerts are enabled', async () => { + it('should install subset of common resources when framework alerts are enabled', async () => { const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); const getClusterClient = jest.fn(() => Promise.resolve(mockClusterClient)); - const getResourceNameMock = jest - .fn() - .mockReturnValueOnce(TECHNICAL_COMPONENT_TEMPLATE_NAME) - .mockReturnValueOnce(ECS_COMPONENT_TEMPLATE_NAME); const installer = new ResourceInstaller({ logger: loggerMock.create(), isWriteEnabled: true, disabledRegistrationContexts: [], - getResourceName: getResourceNameMock, + getResourceName: jest.fn(), getClusterClient, areFrameworkAlertsEnabled: true, pluginStop$, @@ -131,15 +121,12 @@ describe('resourceInstaller', () => { // ILM policy should be handled by framework expect(mockClusterClient.ilm.putLifecycle).not.toHaveBeenCalled(); - expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(2); + // ECS component template should be handled by framework + expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenCalledTimes(1); expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenNthCalledWith( 1, expect.objectContaining({ name: TECHNICAL_COMPONENT_TEMPLATE_NAME }) ); - expect(mockClusterClient.cluster.putComponentTemplate).toHaveBeenNthCalledWith( - 2, - expect.objectContaining({ name: ECS_COMPONENT_TEMPLATE_NAME }) - ); }); it('should install index level resources', async () => { const mockClusterClient = elasticsearchServiceMock.createElasticsearchClient(); diff --git a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts index 6af288e57a4a03..59c74b81712d8f 100644 --- a/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts +++ b/x-pack/plugins/rule_registry/server/rule_data_plugin_service/resource_installer.ts @@ -15,18 +15,16 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { DEFAULT_ALERTS_ILM_POLICY, DEFAULT_ALERTS_ILM_POLICY_NAME, -} from '@kbn/alerting-plugin/server'; -import { ECS_COMPONENT_TEMPLATE_NAME, - TECHNICAL_COMPONENT_TEMPLATE_NAME, -} from '../../common/assets'; +} from '@kbn/alerting-plugin/server'; +import { TECHNICAL_COMPONENT_TEMPLATE_NAME } from '../../common/assets'; import { technicalComponentTemplate } from '../../common/assets/component_templates/technical_component_template'; import { ecsComponentTemplate } from '../../common/assets/component_templates/ecs_component_template'; import type { IndexInfo } from './index_info'; const INSTALLATION_TIMEOUT = 20 * 60 * 1000; // 20 minutes -const TOTAL_FIELDS_LIMIT = 1900; +const TOTAL_FIELDS_LIMIT = 2500; interface ConstructorOptions { getResourceName(relativeName: string): string; getClusterClient: () => Promise; @@ -98,7 +96,7 @@ export class ResourceInstaller { */ public async installCommonResources(): Promise { await this.installWithTimeout('common resources shared between all indices', async () => { - const { getResourceName, logger, areFrameworkAlertsEnabled } = this.options; + const { logger, areFrameworkAlertsEnabled } = this.options; try { // We can install them in parallel @@ -112,16 +110,15 @@ export class ResourceInstaller { name: DEFAULT_ALERTS_ILM_POLICY_NAME, body: DEFAULT_ALERTS_ILM_POLICY, }), + this.createOrUpdateComponentTemplate({ + name: ECS_COMPONENT_TEMPLATE_NAME, + body: ecsComponentTemplate, + }), ]), this.createOrUpdateComponentTemplate({ - name: getResourceName(TECHNICAL_COMPONENT_TEMPLATE_NAME), + name: TECHNICAL_COMPONENT_TEMPLATE_NAME, body: technicalComponentTemplate, }), - - this.createOrUpdateComponentTemplate({ - name: getResourceName(ECS_COMPONENT_TEMPLATE_NAME), - body: ecsComponentTemplate, - }), ]); } catch (err) { logger.error( @@ -315,7 +312,7 @@ export class ResourceInstaller { } private async installNamespacedIndexTemplate(indexInfo: IndexInfo, namespace: string) { - const { logger, getResourceName } = this.options; + const { logger } = this.options; const { componentTemplateRefs, componentTemplates, @@ -329,8 +326,7 @@ export class ResourceInstaller { logger.debug(`Installing index template for ${primaryNamespacedAlias}`); - const technicalComponentNames = [getResourceName(TECHNICAL_COMPONENT_TEMPLATE_NAME)]; - const referencedComponentNames = componentTemplateRefs.map((ref) => getResourceName(ref)); + const technicalComponentNames = [TECHNICAL_COMPONENT_TEMPLATE_NAME]; const ownComponentNames = componentTemplates.map((template) => indexInfo.getComponentTemplateName(template.name) ); @@ -365,11 +361,7 @@ export class ResourceInstaller { // - then we include own component templates registered with this index // - finally, we include technical component templates to make sure the index gets all the // mappings and settings required by all Kibana plugins using rule registry to work properly - composed_of: [ - ...referencedComponentNames, - ...ownComponentNames, - ...technicalComponentNames, - ], + composed_of: [...componentTemplateRefs, ...ownComponentNames, ...technicalComponentNames], template: { settings: { diff --git a/x-pack/plugins/rule_registry/tsconfig.json b/x-pack/plugins/rule_registry/tsconfig.json index a3a2a6d373b2bf..1bb9b96e6aa921 100644 --- a/x-pack/plugins/rule_registry/tsconfig.json +++ b/x-pack/plugins/rule_registry/tsconfig.json @@ -16,7 +16,6 @@ "@kbn/data-plugin", "@kbn/alerting-plugin", "@kbn/security-plugin", - "@kbn/safer-lodash-set", "@kbn/rule-data-utils", "@kbn/es-query", "@kbn/data-views-plugin", @@ -32,6 +31,7 @@ "@kbn/logging", "@kbn/securitysolution-io-ts-utils", "@kbn/share-plugin", + "@kbn/alerts-as-data-utils", ], "exclude": [ "target/**/*", diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index db7a32670727be..4a391ebcab4be2 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -21,10 +21,10 @@ import type { Logger } from '@kbn/core/server'; import { SavedObjectsClient } from '@kbn/core/server'; import type { UsageCounter } from '@kbn/usage-collection-plugin/server'; -import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/rule-registry-plugin/common/assets'; -import type { FieldMap } from '@kbn/rule-registry-plugin/common/field_map'; +import { ECS_COMPONENT_TEMPLATE_NAME } from '@kbn/alerting-plugin/server'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; +import type { FieldMap } from '@kbn/alerts-as-data-utils'; import { technicalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/technical_rule_field_map'; -import { mappingFromFieldMap } from '@kbn/rule-registry-plugin/common/mapping_from_field_map'; import type { IRuleDataClient } from '@kbn/rule-registry-plugin/server'; import { Dataset } from '@kbn/rule-registry-plugin/server'; import type { ListPluginSetup } from '@kbn/lists-plugin/server'; @@ -220,6 +220,7 @@ export class Plugin implements ISecuritySolutionPlugin { Object.entries(aadFieldConversion).forEach(([key, value]) => { aliasesFieldMap[key] = { type: 'alias', + required: false, path: value, }; }); diff --git a/x-pack/plugins/security_solution/tsconfig.json b/x-pack/plugins/security_solution/tsconfig.json index 6f831e7633dbb6..dada95f662bbd3 100644 --- a/x-pack/plugins/security_solution/tsconfig.json +++ b/x-pack/plugins/security_solution/tsconfig.json @@ -141,5 +141,6 @@ "@kbn/securitysolution-ecs", "@kbn/cell-actions", "@kbn/shared-ux-router", + "@kbn/alerts-as-data-utils", ] } diff --git a/x-pack/plugins/synthetics/common/rules/uptime_rule_field_map.ts b/x-pack/plugins/synthetics/common/rules/uptime_rule_field_map.ts index ff69d3a5e6e7fa..be097ed8d8268c 100644 --- a/x-pack/plugins/synthetics/common/rules/uptime_rule_field_map.ts +++ b/x-pack/plugins/synthetics/common/rules/uptime_rule_field_map.ts @@ -9,48 +9,62 @@ export const uptimeRuleFieldMap = { // common fields 'monitor.id': { type: 'keyword', + required: false, }, 'url.full': { type: 'keyword', + required: false, }, 'observer.geo.name': { type: 'keyword', + required: false, }, // monitor status alert fields 'error.message': { type: 'text', + required: false, }, 'agent.name': { type: 'keyword', + required: false, }, 'monitor.name': { type: 'keyword', + required: false, }, 'monitor.type': { type: 'keyword', + required: false, }, // tls alert fields 'tls.server.x509.issuer.common_name': { type: 'keyword', + required: false, }, 'tls.server.x509.subject.common_name': { type: 'keyword', + required: false, }, 'tls.server.x509.not_after': { type: 'date', + required: false, }, 'tls.server.x509.not_before': { type: 'date', + required: false, }, 'tls.server.hash.sha256': { type: 'keyword', + required: false, }, // anomaly alert fields 'anomaly.start': { type: 'date', + required: false, }, 'anomaly.bucket_span.minutes': { type: 'keyword', + required: false, }, } as const; diff --git a/x-pack/plugins/synthetics/server/plugin.ts b/x-pack/plugins/synthetics/server/plugin.ts index 598fdd18b229be..c6120c70c38181 100644 --- a/x-pack/plugins/synthetics/server/plugin.ts +++ b/x-pack/plugins/synthetics/server/plugin.ts @@ -13,7 +13,7 @@ import { SavedObjectsClient, SavedObjectsClientContract, } from '@kbn/core/server'; -import { mappingFromFieldMap } from '@kbn/rule-registry-plugin/common/mapping_from_field_map'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { experimentalRuleFieldMap } from '@kbn/rule-registry-plugin/common/assets/field_maps/experimental_rule_field_map'; import { Dataset } from '@kbn/rule-registry-plugin/server'; import { SyntheticsMonitorClient } from './synthetics_service/synthetics_monitor/synthetics_monitor_client'; diff --git a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data.ts b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data.ts index 3ac2fdb93add8d..ad2c33b079b0aa 100644 --- a/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data.ts +++ b/x-pack/test/alerting_api_integration/spaces_only/tests/alerting/group4/alerts_as_data.ts @@ -5,20 +5,24 @@ * 2.0. */ -import { alertFieldMap } from '@kbn/alerting-plugin/common/alert_schema'; -import { mappingFromFieldMap } from '@kbn/alerting-plugin/common/alert_schema/field_maps/mapping_from_field_map'; +import { alertFieldMap, ecsFieldMap, legacyAlertFieldMap } from '@kbn/alerts-as-data-utils'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import expect from '@kbn/expect'; import { FtrProviderContext } from '../../../../common/ftr_provider_context'; // eslint-disable-next-line import/no-default-export export default function createAlertsAsDataTest({ getService }: FtrProviderContext) { const es = getService('es'); - const commonFrameworkMappings = mappingFromFieldMap(alertFieldMap, 'strict'); + const frameworkMappings = mappingFromFieldMap(alertFieldMap, 'strict'); + const legacyAlertMappings = mappingFromFieldMap(legacyAlertFieldMap, 'strict'); + const ecsMappings = mappingFromFieldMap(ecsFieldMap, 'strict'); describe('alerts as data', () => { it('should install common alerts as data resources on startup', async () => { const ilmPolicyName = '.alerts-ilm-policy'; - const componentTemplateName = 'alerts-common-component-template'; + const frameworkComponentTemplateName = '.alerts-framework-mappings'; + const legacyComponentTemplateName = '.alerts-legacy-alert-mappings'; + const ecsComponentTemplateName = '.alerts-ecs-mappings'; const commonIlmPolicy = await es.ilm.getLifecycle({ name: ilmPolicyName, @@ -41,23 +45,65 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex }, }); - const { component_templates: componentTemplates } = await es.cluster.getComponentTemplate({ - name: componentTemplateName, + const { component_templates: componentTemplates1 } = await es.cluster.getComponentTemplate({ + name: frameworkComponentTemplateName, }); - expect(componentTemplates.length).to.eql(1); - const commonComponentTemplate = componentTemplates[0]; + expect(componentTemplates1.length).to.eql(1); + const frameworkComponentTemplate = componentTemplates1[0]; + + expect(frameworkComponentTemplate.name).to.eql(frameworkComponentTemplateName); + expect(frameworkComponentTemplate.component_template.template.mappings).to.eql( + frameworkMappings + ); + expect(frameworkComponentTemplate.component_template.template.settings).to.eql({ + index: { + number_of_shards: 1, + mapping: { + total_fields: { + limit: 1500, + }, + }, + }, + }); + + const { component_templates: componentTemplates2 } = await es.cluster.getComponentTemplate({ + name: legacyComponentTemplateName, + }); + + expect(componentTemplates2.length).to.eql(1); + const legacyComponentTemplate = componentTemplates2[0]; - expect(commonComponentTemplate.name).to.eql(componentTemplateName); - expect(commonComponentTemplate.component_template.template.mappings).to.eql( - commonFrameworkMappings + expect(legacyComponentTemplate.name).to.eql(legacyComponentTemplateName); + expect(legacyComponentTemplate.component_template.template.mappings).to.eql( + legacyAlertMappings ); - expect(commonComponentTemplate.component_template.template.settings).to.eql({ + expect(legacyComponentTemplate.component_template.template.settings).to.eql({ + index: { + number_of_shards: 1, + mapping: { + total_fields: { + limit: 1500, + }, + }, + }, + }); + + const { component_templates: componentTemplates3 } = await es.cluster.getComponentTemplate({ + name: ecsComponentTemplateName, + }); + + expect(componentTemplates3.length).to.eql(1); + const ecsComponentTemplate = componentTemplates3[0]; + + expect(ecsComponentTemplate.name).to.eql(ecsComponentTemplateName); + expect(ecsComponentTemplate.component_template.template.mappings).to.eql(ecsMappings); + expect(ecsComponentTemplate.component_template.template.settings).to.eql({ index: { number_of_shards: 1, mapping: { total_fields: { - limit: 100, + limit: 2500, }, }, }, @@ -65,7 +111,7 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex }); it('should install context specific alerts as data resources on startup', async () => { - const componentTemplateName = 'alerts-test.always-firing-component-template'; + const componentTemplateName = '.alerts-test.always-firing-mappings'; const indexTemplateName = '.alerts-test.always-firing-default-template'; const indexName = '.alerts-test.always-firing-default-000001'; const contextSpecificMappings = { @@ -98,7 +144,7 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex number_of_shards: 1, mapping: { total_fields: { - limit: 100, + limit: 1500, }, }, }, @@ -114,8 +160,8 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex '.alerts-test.always-firing-default-*', ]); expect(contextIndexTemplate.index_template.composed_of).to.eql([ - 'alerts-common-component-template', - 'alerts-test.always-firing-component-template', + '.alerts-test.always-firing-mappings', + '.alerts-framework-mappings', ]); expect(contextIndexTemplate.index_template.template!.mappings).to.eql({ dynamic: false, @@ -150,7 +196,7 @@ export default function createAlertsAsDataTest({ getService }: FtrProviderContex dynamic: 'false', properties: { ...contextSpecificMappings, - ...commonFrameworkMappings.properties, + ...frameworkMappings.properties, }, }); diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts b/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts index 02765aa8c2a772..b62cf7b39965c6 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/get_summarized_alerts.ts @@ -10,7 +10,7 @@ import type { ElasticsearchClient, Logger, LogMeta } from '@kbn/core/server'; import sinon from 'sinon'; import { v4 as uuidv4 } from 'uuid'; import expect from '@kbn/expect'; -import { mappingFromFieldMap } from '@kbn/rule-registry-plugin/common/mapping_from_field_map'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { AlertConsumers, ALERT_REASON, diff --git a/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts b/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts index dc5752417bfb49..10351fc6cf2efb 100644 --- a/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts +++ b/x-pack/test/rule_registry/spaces_only/tests/trial/lifecycle_executor.ts @@ -9,7 +9,7 @@ import { type Subject, ReplaySubject } from 'rxjs'; import type { ElasticsearchClient, Logger, LogMeta } from '@kbn/core/server'; import sinon from 'sinon'; import expect from '@kbn/expect'; -import { mappingFromFieldMap } from '@kbn/rule-registry-plugin/common/mapping_from_field_map'; +import { mappingFromFieldMap } from '@kbn/alerting-plugin/common'; import { AlertConsumers, ALERT_REASON, diff --git a/x-pack/test/tsconfig.json b/x-pack/test/tsconfig.json index 476b6223b95914..be736ec73fd0da 100644 --- a/x-pack/test/tsconfig.json +++ b/x-pack/test/tsconfig.json @@ -115,6 +115,7 @@ "@kbn/cloud-security-posture-plugin", "@kbn/cloud-integration-saml-provider-plugin", "@kbn/security-api-integration-helpers", + "@kbn/alerts-as-data-utils", "@kbn/discover-plugin", ] } diff --git a/yarn.lock b/yarn.lock index 905ce3e6a3b5dc..64d6edc4884c21 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2785,6 +2785,10 @@ version "0.0.0" uid "" +"@kbn/alerts-as-data-utils@link:packages/kbn-alerts-as-data-utils": + version "0.0.0" + uid "" + "@kbn/alerts-restricted-fixtures-plugin@link:x-pack/test/alerting_api_integration/common/plugins/alerts_restricted": version "0.0.0" uid ""