diff --git a/x-pack/plugins/security_solution/common/constants.ts b/x-pack/plugins/security_solution/common/constants.ts index cc7e8df757c1d..686c05e2fda51 100644 --- a/x-pack/plugins/security_solution/common/constants.ts +++ b/x-pack/plugins/security_solution/common/constants.ts @@ -136,9 +136,13 @@ export const TIMELINE_PREPACKAGED_URL = `${TIMELINE_URL}/_prepackaged`; * Default signals index key for kibana.dev.yml */ export const SIGNALS_INDEX_KEY = 'signalsIndex'; + export const DETECTION_ENGINE_SIGNALS_URL = `${DETECTION_ENGINE_URL}/signals`; export const DETECTION_ENGINE_SIGNALS_STATUS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/status`; export const DETECTION_ENGINE_QUERY_SIGNALS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/search`; +export const DETECTION_ENGINE_SIGNALS_MIGRATION_URL = `${DETECTION_ENGINE_SIGNALS_URL}/migration`; +export const DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL = `${DETECTION_ENGINE_SIGNALS_URL}/migration_status`; +export const DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL = `${DETECTION_ENGINE_SIGNALS_URL}/finalize_migration`; /** * Common naming convention for an unauthenticated user diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_signals_migration_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_signals_migration_schema.mock.ts new file mode 100644 index 0000000000000..58e50f84366e3 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_signals_migration_schema.mock.ts @@ -0,0 +1,13 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { CreateSignalsMigrationSchema } from './create_signals_migration_schema'; + +export const getCreateSignalsMigrationSchemaMock = ( + index: string = 'signals-index' +): CreateSignalsMigrationSchema => ({ + index: [index], +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_signals_migration_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_signals_migration_schema.ts new file mode 100644 index 0000000000000..2c441bd31fe2c --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/create_signals_migration_schema.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +import { index } from '../common/schemas'; +import { PositiveInteger, PositiveIntegerGreaterThanZero } from '../types'; + +export const signalsReindexOptions = t.partial({ + requests_per_second: t.number, + size: PositiveIntegerGreaterThanZero, + slices: PositiveInteger, +}); + +export type SignalsReindexOptions = t.TypeOf; + +export const createSignalsMigrationSchema = t.intersection([ + t.exact( + t.type({ + index, + }) + ), + t.exact(signalsReindexOptions), +]); + +export type CreateSignalsMigrationSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.mock.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.mock.ts new file mode 100644 index 0000000000000..d0387586a2527 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.mock.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { FinalizeSignalsMigrationSchema } from './finalize_signals_migration_schema'; + +export const getFinalizeSignalsMigrationSchemaMock = (): FinalizeSignalsMigrationSchema => ({ + migration_token: + 'eyJkZXN0aW5hdGlvbkluZGV4IjoiZGVzdGluYXRpb25JbmRleCIsInNvdXJjZUluZGV4Ijoic291cmNlSW5kZXgiLCJ0YXNrSWQiOiJteS10YXNrLWlkIn0=', +}); diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.ts new file mode 100644 index 0000000000000..7ab2ee3810258 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/finalize_signals_migration_schema.ts @@ -0,0 +1,19 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +import { NonEmptyString } from '../types'; + +const migrationToken = NonEmptyString; + +export const finalizeSignalsMigrationSchema = t.exact( + t.type({ + migration_token: migrationToken, + }) +); + +export type FinalizeSignalsMigrationSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/common/detection_engine/schemas/request/get_migration_status_schema.ts b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/get_migration_status_schema.ts new file mode 100644 index 0000000000000..dfa230fc21d71 --- /dev/null +++ b/x-pack/plugins/security_solution/common/detection_engine/schemas/request/get_migration_status_schema.ts @@ -0,0 +1,17 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import * as t from 'io-ts'; + +import { from } from '../common/schemas'; + +export const getMigrationStatusSchema = t.exact( + t.type({ + from, + }) +); + +export type GetMigrationStatusSchema = t.TypeOf; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_aliases.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_aliases.ts new file mode 100644 index 0000000000000..f2a5d201b62a6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_aliases.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; + +interface AliasesResponse { + [indexName: string]: { + aliases: { + [aliasName: string]: { + is_write_index: boolean; + }; + }; + }; +} + +interface IndexAlias { + alias: string; + index: string; + isWriteIndex: boolean; +} + +/** + * Retrieves all index aliases for a given alias name + * + * @param esClient An {@link ElasticsearchClient} + * @param alias alias name used to filter results + * + * @returns an array of {@link IndexAlias} objects + */ +export const getIndexAliases = async ({ + esClient, + alias, +}: { + esClient: ElasticsearchClient; + alias: string; +}): Promise => { + const response = await esClient.indices.getAlias({ + name: alias, + }); + + return Object.keys(response.body).map((index) => ({ + alias, + index, + isWriteIndex: response.body[index].aliases[alias]?.is_write_index === true, + })); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_count.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_count.ts new file mode 100644 index 0000000000000..8786c9eb1d857 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/index/get_index_count.ts @@ -0,0 +1,29 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; + +/** + * Retrieves the count of documents in a given index + * + * @param esClient An {@link ElasticsearchClient} + * @param index index whose documents will be counted + * + * @returns the document count + */ +export const getIndexCount = async ({ + esClient, + index, +}: { + esClient: ElasticsearchClient; + index: string; +}): Promise => { + const response = await esClient.count<{ count: number }>({ + index, + }); + + return response.body.count; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.test.ts new file mode 100644 index 0000000000000..b638e89436601 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.test.ts @@ -0,0 +1,25 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { createSignalsMigrationIndex } from './create_signals_migration_index'; + +describe('getMigrationStatus', () => { + let esClient: ElasticsearchClient; + + beforeEach(() => { + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('creates an index suffixed with the template version', async () => { + await createSignalsMigrationIndex({ esClient, index: 'my-signals-index', version: 4 }); + + expect(esClient.indices.create).toHaveBeenCalledWith( + expect.objectContaining({ index: 'my-signals-index-r000004' }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.ts new file mode 100644 index 0000000000000..17929e39c24b7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/create_signals_migration_index.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; + +/** + * Creates the destination index to be used during the migration of a + * given signals index. + * + * The destination index's name is determined by adding a suffix of + * `-r${templateVersion}` to the source index name + * + * @param esClient An {@link ElasticsearchClient} + * @param index name of the source signals index + * @param version version of the current signals template/mappings + * + * @returns the name of the created index + */ +export const createSignalsMigrationIndex = async ({ + esClient, + index, + version, +}: { + esClient: ElasticsearchClient; + index: string; + version: number; +}): Promise => { + const paddedVersion = `${version}`.padStart(6, '0'); + const destinationIndexName = `${index}-r${paddedVersion}`; + + const response = await esClient.indices.create<{ index: string }>({ + index: destinationIndexName, + body: { + settings: { + index: { + lifecycle: { + indexing_complete: true, + }, + }, + }, + }, + }); + + return response.body.index; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.mock.ts new file mode 100644 index 0000000000000..08b74b6c2ca7b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.mock.ts @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IndexMappingsResponse, MigrationStatusSearchResponse } from './types'; + +export const getMigrationStatusSearchResponseMock = ( + indices: string[] = ['signals-index'], + signalVersions: number[] = [-1] +): MigrationStatusSearchResponse => ({ + aggregations: { + signals_indices: { + buckets: indices.map((index) => ({ + key: index, + signal_versions: { + buckets: signalVersions.map((version) => ({ + key: version, + doc_count: 4, + })), + }, + })), + }, + }, +}); + +export const getIndexMappingsResponseMock = ( + index: string = 'signals-index' +): IndexMappingsResponse => ({ + [index]: { mappings: { _meta: { version: -1 } } }, +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.test.ts new file mode 100644 index 0000000000000..2cd506fe1e870 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.test.ts @@ -0,0 +1,103 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { + getIndexMappingsResponseMock, + getMigrationStatusSearchResponseMock, +} from './get_migration_status.mock'; +import { getMigrationStatus } from './get_migration_status'; + +describe('getMigrationStatus', () => { + let esClient: ElasticsearchClient; + + beforeEach(() => { + esClient = elasticsearchServiceMock.createElasticsearchClient(); + + // mock index version + (esClient.indices.getMapping as jest.Mock).mockResolvedValue({ + body: { + ...getIndexMappingsResponseMock('index1'), + }, + }); + + // mock index search + (esClient.search as jest.Mock).mockResolvedValue({ + body: { + ...getMigrationStatusSearchResponseMock(['index1']), + }, + }); + }); + + it('returns one entry for each index provided', async () => { + (esClient.indices.getMapping as jest.Mock).mockResolvedValueOnce({ + body: { + ...getIndexMappingsResponseMock('index1'), + ...getIndexMappingsResponseMock('index2'), + ...getIndexMappingsResponseMock('index3'), + }, + }); + + // mock index search + (esClient.search as jest.Mock).mockResolvedValueOnce({ + body: getMigrationStatusSearchResponseMock(['index1', 'index2', 'index3']), + }); + + const migrationStatuses = await getMigrationStatus({ + esClient, + index: ['index1', 'index2', 'index3'], + }); + + expect(migrationStatuses).toHaveLength(3); + }); + + it('returns the name and version for each index provided', async () => { + const [migrationStatus] = await getMigrationStatus({ + esClient, + index: ['index1'], + }); + + expect(migrationStatus).toEqual( + expect.objectContaining({ + name: 'index1', + version: -1, + }) + ); + }); + + it('returns the breakdown of signals versions available in each index', async () => { + const [migrationStatus] = await getMigrationStatus({ + esClient, + index: ['index1'], + }); + + expect(migrationStatus).toEqual( + expect.objectContaining({ + signal_versions: [{ key: -1, doc_count: 4 }], + }) + ); + }); + + it('defaults the index version to 0 if missing from the mapping', async () => { + (esClient.indices.getMapping as jest.Mock).mockResolvedValueOnce({ + body: { + index1: { mappings: {} }, + }, + }); + + const [migrationStatus] = await getMigrationStatus({ + esClient, + index: ['index1'], + }); + + expect(migrationStatus).toEqual( + expect.objectContaining({ + version: 0, + }) + ); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.ts new file mode 100644 index 0000000000000..af0a28e807fa2 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_migration_status.ts @@ -0,0 +1,76 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; +import { IndexMappingsResponse, MigrationStatus, MigrationStatusSearchResponse } from './types'; + +/** + * Retrieves a breakdown of information relevant to the migration of each + * given signals index. + * + * This includes: + * * the mappings version of the index + * * aggregated counts of the schema versions of signals in the index + * * aggregated counts of the migration versions of signals in the index + * + * @param esClient An {@link ElasticsearchClient} + * @param index name(s) of the signals index(es) + * + * @returns an array of {@link MigrationStatus} objects + * + * @throws if elasticsearch returns an error + */ +export const getMigrationStatus = async ({ + esClient, + index, +}: { + esClient: ElasticsearchClient; + index: string[]; +}): Promise => { + if (index.length === 0) { + return []; + } + + const { body: indexVersions } = await esClient.indices.getMapping({ + index, + }); + const response = await esClient.search({ + index, + size: 0, + body: { + aggs: { + signals_indices: { + terms: { + field: '_index', + }, + aggs: { + signal_versions: { + terms: { + field: 'signal._meta.version', + missing: 0, + }, + }, + }, + }, + }, + }, + }); + + const indexBuckets = response.body.aggregations.signals_indices.buckets; + return indexBuckets.reduce((statuses, bucket) => { + const indexName = bucket.key; + const indexVersion = indexVersions[indexName]?.mappings?._meta?.version ?? 0; + + return [ + ...statuses, + { + name: indexName, + version: indexVersion, + signal_versions: bucket.signal_versions.buckets, + }, + ]; + }, []); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_signals_indices_in_range.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_signals_indices_in_range.test.ts new file mode 100644 index 0000000000000..a940544a92693 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_signals_indices_in_range.test.ts @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; +import { elasticsearchServiceMock } from 'src/core/server/mocks'; +import { getSignalsIndicesInRange } from './get_signals_indices_in_range'; + +describe('getSignalsIndicesInRange', () => { + let esClient: ElasticsearchClient; + + beforeEach(() => { + esClient = elasticsearchServiceMock.createElasticsearchClient(); + }); + + it('returns empty array if provided index is empty', async () => { + const indicesInRange = await getSignalsIndicesInRange({ esClient, index: [], from: 'now-3d' }); + expect(indicesInRange).toEqual([]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_signals_indices_in_range.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_signals_indices_in_range.ts new file mode 100644 index 0000000000000..9ef56f8741811 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/get_signals_indices_in_range.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; + +interface IndexesResponse { + aggregations: { + indexes: { + buckets: Array<{ + key: string; + }>; + }; + }; +} + +/** + * Retrieves the list of indices containing signals that fall between now and + * the given date. This is most relevant to signals migrations, where we want + * to scope the number of indexes/documents that we migrate. + * + * + * @param esClient An {@link ElasticsearchClient} + * @param from date math string representing the start of the range + * @param index name(s) of the signals index(es) + * + * @returns an array of index names + */ +export const getSignalsIndicesInRange = async ({ + esClient, + from, + index, +}: { + esClient: ElasticsearchClient; + index: string[]; + from: string; +}): Promise => { + if (index.length === 0) { + return []; + } + + const response = await esClient.search({ + index, + body: { + aggs: { + indexes: { + terms: { + field: '_index', + }, + }, + }, + query: { + bool: { + filter: [ + { + range: { + '@timestamp': { + gte: from, + lte: 'now', + format: 'strict_date_optional_time', + }, + }, + }, + ], + }, + }, + size: 0, + }, + }); + + return response.body.aggregations.indexes.buckets.map((bucket) => bucket.key); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/helpers.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/helpers.test.ts new file mode 100644 index 0000000000000..0a7f714553fcd --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/helpers.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { MigrationDetails } from './types'; +import { decodeMigrationToken, encodeMigrationToken } from './helpers'; + +describe('migration tokens', () => { + let details: MigrationDetails; + + beforeEach(() => { + details = { + destinationIndex: 'destinationIndex', + sourceIndex: 'sourceIndex', + taskId: 'my-task-id', + }; + }); + + describe('decodeMigrationToken', () => { + it('decodes a valid token to migration details', () => { + const token = encodeMigrationToken({ ...details }); + const decodedDetails = decodeMigrationToken(token); + expect(decodedDetails).toEqual(details); + }); + + it('decoding a misencoded string throws an error', () => { + const badToken = 'not-properly-encoded'; + expect(() => decodeMigrationToken(badToken)).toThrowError( + 'An error occurred while decoding the migration token: [not-properly-encoded]' + ); + }); + + it('decoding invalid details throws an error', () => { + const invalidDetails = ({ ...details, taskId: null } as unknown) as MigrationDetails; + const token = encodeMigrationToken(invalidDetails); + expect(() => decodeMigrationToken(token)).toThrowError( + 'An error occurred while decoding the migration token: [eyJkZXN0aW5hdGlvbkluZGV4IjoiZGVzdGluYXRpb25JbmRleCIsInNvdXJjZUluZGV4Ijoic291cmNlSW5kZXgiLCJ0YXNrSWQiOm51bGx9]' + ); + }); + }); + + describe('encodeMigrationToken', () => { + it('encodes idempotently', () => { + expect(encodeMigrationToken(details)).toEqual(encodeMigrationToken(details)); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/helpers.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/helpers.ts new file mode 100644 index 0000000000000..10763e0f3f41c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/helpers.ts @@ -0,0 +1,62 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { BadRequestError } from '../errors/bad_request_error'; +import { MigrationDetails, MigrationStatus } from './types'; + +const decodeBase64 = (base64: string) => Buffer.from(base64, 'base64').toString('utf8'); +const encodeBase64 = (utf8: string) => Buffer.from(utf8, 'utf8').toString('base64'); + +export const encodeMigrationToken = (details: MigrationDetails): string => + encodeBase64(JSON.stringify(details)); + +export const decodeMigrationToken = (token: string): MigrationDetails => { + try { + const details = JSON.parse(decodeBase64(token)) as MigrationDetails; + + if (details.destinationIndex == null || details.sourceIndex == null || details.taskId == null) { + throw new TypeError(); + } + + return details; + } catch (_) { + throw new BadRequestError(`An error occurred while decoding the migration token: [${token}]`); + } +}; + +export const isOutdated = ({ current, target }: { current: number; target: number }): boolean => + current < target; + +const mappingsAreOutdated = ({ + status, + version, +}: { + status: MigrationStatus; + version: number; +}): boolean => isOutdated({ current: status.version, target: version }); + +const signalsAreOutdated = ({ + status, + version, +}: { + status: MigrationStatus; + version: number; +}): boolean => + status.signal_versions.some((signalVersion) => { + return ( + signalVersion.doc_count > 0 && isOutdated({ current: signalVersion.key, target: version }) + ); + }); + +export const indexIsOutdated = ({ + status, + version, +}: { + status?: MigrationStatus; + version: number; +}): boolean => + status != null && + (mappingsAreOutdated({ status, version }) || signalsAreOutdated({ status, version })); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migrate_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migrate_signals.ts new file mode 100644 index 0000000000000..98f492a0b80fe --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migrate_signals.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; +import { SignalsReindexOptions } from '../../../../common/detection_engine/schemas/request/create_signals_migration_schema'; +import { createSignalsMigrationIndex } from './create_signals_migration_index'; +import { MigrationDetails } from './types'; + +/** + * Migrates signals for a given concrete index. Signals are reindexed into a + * new index in order to receive new fields. Migrated signals have a + * `signal._meta.version` field representing the mappings version at the time of the migration. + * + * @param esClient An {@link ElasticsearchClient} + * @param index name of the concrete signals index to be migrated + * @param version version of the current signals template/mappings + * @param reindexOptions object containing reindex options {@link SignalsReindexOptions} + * + * @returns identifying information representing the {@link MigrationDetails} + * @throws if elasticsearch returns an error + */ +export const migrateSignals = async ({ + esClient, + index, + reindexOptions, + version, +}: { + esClient: ElasticsearchClient; + index: string; + reindexOptions: SignalsReindexOptions; + version: number; +}): Promise => { + const migrationIndex = await createSignalsMigrationIndex({ + esClient, + index, + version, + }); + + const { size, ...reindexQueryOptions } = reindexOptions; + + const response = await esClient.reindex<{ task: string }>({ + body: { + dest: { index: migrationIndex }, + source: { index, size }, + script: { + lang: 'painless', + source: ` + if (ctx._source.signal._meta == null) { + ctx._source.signal._meta = [:]; + } + ctx._source.signal._meta.version = params.version; + `, + params: { + version, + }, + }, + }, + ...reindexQueryOptions, + refresh: true, + wait_for_completion: false, + }); + + return { + destinationIndex: migrationIndex, + sourceIndex: index, + taskId: response.body.task, + }; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_cleanup.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_cleanup.ts new file mode 100644 index 0000000000000..4bf3dbd9c02cf --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_cleanup.ts @@ -0,0 +1,95 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; +import migrationCleanupPolicy from './migration_cleanup_policy.json'; + +export const getMigrationCleanupPolicyName = (alias: string): string => + `${alias}-migration-cleanup`; + +const getPolicyExists = async ({ + esClient, + policy, +}: { + esClient: ElasticsearchClient; + policy: string; +}): Promise => { + try { + await esClient.ilm.getLifecycle({ + policy, + }); + return true; + } catch (err) { + if (err.statusCode === 404) { + return false; + } else { + throw err; + } + } +}; + +/** + * Checks that the migration cleanup ILM policy exists for the given signals + * alias, and creates it if necessary. + * + * This policy is applied to outdated signals indexes post-migration, ensuring + * that they are eventually deleted. + * + * @param esClient An {@link ElasticsearchClient} + * @param alias name of the signals alias + * + * @throws if elasticsearch returns an error + */ +export const ensureMigrationCleanupPolicy = async ({ + esClient, + alias, +}: { + esClient: ElasticsearchClient; + alias: string; +}): Promise => { + const policy = getMigrationCleanupPolicyName(alias); + + const policyExists = await getPolicyExists({ esClient, policy }); + if (!policyExists) { + await esClient.ilm.putLifecycle({ + policy, + body: migrationCleanupPolicy, + }); + } +}; + +/** + * Applies the migration cleanup ILM policy to the specified signals index. + * + * This is invoked for an outdated signals index after a successful index + * migration, ensuring that it's eventually deleted. + * + * @param esClient An {@link ElasticsearchClient} + * @param alias name of the signals alias + * @param index name of the concrete signals index to receive the policy + * + * @throws if elasticsearch returns an error + */ +export const applyMigrationCleanupPolicy = async ({ + alias, + esClient, + index, +}: { + alias: string; + esClient: ElasticsearchClient; + index: string; +}): Promise => { + await esClient.indices.putSettings({ + index, + body: { + index: { + lifecycle: { + name: getMigrationCleanupPolicyName(alias), + }, + }, + }, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_cleanup_policy.json b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_cleanup_policy.json new file mode 100644 index 0000000000000..a554f47591671 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/migration_cleanup_policy.json @@ -0,0 +1,21 @@ +{ + "policy": { + "phases": { + "hot": { + "min_age": "0ms", + "actions": { + "rollover": { + "max_size": "50gb", + "max_age": "30d" + } + } + }, + "delete": { + "min_age": "30d", + "actions": { + "delete": {} + } + } + } + } +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/replace_signals_index_alias.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/replace_signals_index_alias.ts new file mode 100644 index 0000000000000..f7e167a727902 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/replace_signals_index_alias.ts @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ElasticsearchClient } from 'src/core/server'; + +/** + * Updates aliases for the old and new concrete indexes specified, respectively + * removing and adding them atomically. + * + * This is invoked as part of the finalization of a signals migration: once the + * migrated index has been verified, its alias replaces the outdated index. + * + * @param esClient An {@link ElasticsearchClient} + * @param alias name of the signals alias + * @param newIndex name of the concrete signals index to be aliased + * @param oldIndex name of the concrete signals index to be unaliased + * + * @throws if elasticsearch returns an error + */ +export const replaceSignalsIndexAlias = async ({ + alias, + esClient, + newIndex, + oldIndex, +}: { + alias: string; + esClient: ElasticsearchClient; + newIndex: string; + oldIndex: string; +}): Promise => { + await esClient.indices.updateAliases({ + body: { + actions: [ + { remove: { index: oldIndex, alias } }, + { add: { index: newIndex, alias, is_write_index: false } }, + ], + }, + }); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/types.ts new file mode 100644 index 0000000000000..0c05361b0941b --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/migrations/types.ts @@ -0,0 +1,39 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface Bucket { + key: number; + doc_count: number; +} + +export interface MigrationStatus { + name: string; + version: number; + signal_versions: Bucket[]; +} + +export interface MigrationDetails { + destinationIndex: string; + sourceIndex: string; + taskId: string; +} + +export interface MigrationStatusSearchResponse { + aggregations: { + signals_indices: { + buckets: Array<{ + key: string; + signal_versions: { + buckets: Bucket[]; + }; + }>; + }; + }; +} + +export interface IndexMappingsResponse { + [indexName: string]: { mappings: { _meta: { version: number } } }; +} diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts index 38ac6372fdb9c..ea95c4fa78842 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/__mocks__/request_responses.ts @@ -15,6 +15,7 @@ import { INTERNAL_RULE_ID_KEY, INTERNAL_IMMUTABLE_KEY, DETECTION_ENGINE_PREPACKAGED_URL, + DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL, } from '../../../../../common/constants'; import { ShardsResponse } from '../../../types'; import { @@ -27,6 +28,7 @@ import { RuleNotificationAlertType } from '../../notifications/types'; import { QuerySignalsSchemaDecoded } from '../../../../../common/detection_engine/schemas/request/query_signals_index_schema'; import { SetSignalsStatusSchemaDecoded } from '../../../../../common/detection_engine/schemas/request/set_signal_status_schema'; import { getCreateRulesSchemaMock } from '../../../../../common/detection_engine/schemas/request/rule_schemas.mock'; +import { getFinalizeSignalsMigrationSchemaMock } from '../../../../../common/detection_engine/schemas/request/finalize_signals_migration_schema.mock'; import { getListArrayMock } from '../../../../../common/detection_engine/schemas/types/lists.mock'; import { EqlSearchResponse } from '../../../../../common/detection_engine/types'; import { getThreatMock } from '../../../../../common/detection_engine/schemas/types/threat.mock'; @@ -648,3 +650,10 @@ export const getFindNotificationsResultWithSingleHit = (): FindHit + requestMock.create({ + method: 'post', + path: DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL, + body: getFinalizeSignalsMigrationSchemaMock(), + }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts index e7618f155967b..fe2976dc166a1 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/check_template_version.ts @@ -4,22 +4,35 @@ * you may not use this file except in compliance with the Elastic License. */ -import { get } from 'lodash'; -import { LegacyAPICaller } from '../../../../../../../../src/core/server'; -import { getTemplateExists } from '../../index/get_template_exists'; +import { ElasticsearchClient } from 'src/core/server'; +import { isOutdated } from '../../migrations/helpers'; import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; -export const templateNeedsUpdate = async (callCluster: LegacyAPICaller, index: string) => { - const templateExists = await getTemplateExists(callCluster, index); - if (!templateExists) { - return true; +export const getTemplateVersion = async ({ + alias, + esClient, +}: { + esClient: ElasticsearchClient; + alias: string; +}): Promise => { + try { + const response = await esClient.indices.getTemplate<{ + [templateName: string]: { version: number }; + }>({ name: alias }); + return response.body[alias].version ?? 0; + } catch (e) { + return 0; } - const existingTemplate: unknown = await callCluster('indices.getTemplate', { - name: index, - }); - const existingTemplateVersion: number | undefined = get(existingTemplate, [index, 'version']); - if (existingTemplateVersion === undefined || existingTemplateVersion < SIGNALS_TEMPLATE_VERSION) { - return true; - } - return false; +}; + +export const templateNeedsUpdate = async ({ + alias, + esClient, +}: { + alias: string; + esClient: ElasticsearchClient; +}): Promise => { + const templateVersion = await getTemplateVersion({ alias, esClient }); + + return isOutdated({ current: templateVersion, target: SIGNALS_TEMPLATE_VERSION }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts index de28d2eee1805..8280e86bdf2cc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/create_index_route.ts @@ -14,9 +14,11 @@ import { setPolicy } from '../../index/set_policy'; import { setTemplate } from '../../index/set_template'; import { getSignalsTemplate, SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; import { createBootstrapIndex } from '../../index/create_bootstrap_index'; +import { ensureMigrationCleanupPolicy } from '../../migrations/migration_cleanup'; import signalsPolicy from './signals_policy.json'; import { templateNeedsUpdate } from './check_template_version'; import { getIndexVersion } from './get_index_version'; +import { isOutdated } from '../../migrations/helpers'; export const createIndexRoute = (router: IRouter) => { router.post( @@ -61,6 +63,7 @@ export const createDetectionIndex = async ( siemClient: AppClient ): Promise => { const clusterClient = context.core.elasticsearch.legacy.client; + const esClient = context.core.elasticsearch.client.asCurrentUser; const callCluster = clusterClient.callAsCurrentUser; if (!siemClient) { @@ -68,17 +71,18 @@ export const createDetectionIndex = async ( } const index = siemClient.getSignalsIndex(); + await ensureMigrationCleanupPolicy({ alias: index, esClient }); const policyExists = await getPolicyExists(callCluster, index); if (!policyExists) { await setPolicy(callCluster, index, signalsPolicy); } - if (await templateNeedsUpdate(callCluster, index)) { + if (await templateNeedsUpdate({ alias: index, esClient })) { await setTemplate(callCluster, index, getSignalsTemplate(index)); } const indexExists = await getIndexExists(callCluster, index); if (indexExists) { const indexVersion = await getIndexVersion(callCluster, index); - if ((indexVersion ?? 0) < SIGNALS_TEMPLATE_VERSION) { + if (isOutdated({ current: indexVersion, target: SIGNALS_TEMPLATE_VERSION })) { await callCluster('indices.rollover', { alias: index }); } } else { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts index 062cffd393555..8553c427588d3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_index_version.ts @@ -21,7 +21,7 @@ interface IndexAliasResponse { export const getIndexVersion = async ( callCluster: LegacyAPICaller, index: string -): Promise => { +): Promise => { const indexAlias: IndicesAliasResponse = await callCluster('indices.getAlias', { index, }); @@ -29,8 +29,8 @@ export const getIndexVersion = async ( (key) => indexAlias[key].aliases[index].is_write_index ); if (writeIndex === undefined) { - return undefined; + return 0; } const writeIndexMapping = await readIndex(callCluster, writeIndex); - return get(writeIndexMapping, [writeIndex, 'mappings', '_meta', 'version']); + return get(writeIndexMapping, [writeIndex, 'mappings', '_meta', 'version']) ?? 0; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts index 664b215549327..55b5a4017398c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template.ts @@ -7,7 +7,7 @@ import signalsMapping from './signals_mapping.json'; import ecsMapping from './ecs_mapping.json'; -export const SIGNALS_TEMPLATE_VERSION = 13; +export const SIGNALS_TEMPLATE_VERSION = 14; export const MIN_EQL_RULE_INDEX_VERSION = 2; export const getSignalsTemplate = (index: string) => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts index 497352b563d36..d898d3fd59240 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/read_index_route.ts @@ -10,6 +10,7 @@ import { transformError, buildSiemResponse } from '../utils'; import { getIndexExists } from '../../index/get_index_exists'; import { SIGNALS_TEMPLATE_VERSION } from './get_signals_template'; import { getIndexVersion } from './get_index_version'; +import { isOutdated } from '../../migrations/helpers'; export const readIndexRoute = (router: IRouter) => { router.get( @@ -38,7 +39,10 @@ export const readIndexRoute = (router: IRouter) => { let mappingOutdated: boolean | null = null; try { const indexVersion = await getIndexVersion(clusterClient.callAsCurrentUser, index); - mappingOutdated = (indexVersion ?? 0) < SIGNALS_TEMPLATE_VERSION; + mappingOutdated = isOutdated({ + current: indexVersion, + target: SIGNALS_TEMPLATE_VERSION, + }); } catch (err) { const error = transformError(err); // Some users may not have the view_index_metadata permission necessary to check the index mapping version diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 890505e9693c4..909264c57067b 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -3,6 +3,13 @@ "properties": { "signal": { "properties": { + "_meta": { + "properties": { + "version": { + "type": "long" + } + } + }, "parent": { "properties": { "rule": { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/create_signals_migration_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/create_signals_migration_route.test.ts new file mode 100644 index 0000000000000..3e3a43855fa46 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/create_signals_migration_route.test.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { requestContextMock, requestMock, serverMock } from '../__mocks__'; +import { createSignalsMigrationRoute } from './create_signals_migration_route'; +import { + getIndexMappingsResponseMock, + getMigrationStatusSearchResponseMock, +} from '../../migrations/get_migration_status.mock'; +import { SignalsReindexOptions } from '../../../../../common/detection_engine/schemas/request/create_signals_migration_schema'; +import { DETECTION_ENGINE_SIGNALS_MIGRATION_URL } from '../../../../../common/constants'; +import { getCreateSignalsMigrationSchemaMock } from '../../../../../common/detection_engine/schemas/request/create_signals_migration_schema.mock'; + +describe('query for signal', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + + beforeEach(() => { + server = serverMock.create(); + ({ clients, context } = requestContextMock.createTools()); + + // @ts-expect-error mocking the bare minimum of our queries + // get our migration status + clients.newClusterClient.asCurrentUser.search.mockResolvedValueOnce({ + body: getMigrationStatusSearchResponseMock(['my-index']), + }); + + // @ts-expect-error mocking the bare minimum of our queries + // get our signals aliases + clients.newClusterClient.asCurrentUser.indices.getAlias.mockResolvedValueOnce({ + body: { 'my-index': { aliases: {} } }, + }); + + // @ts-expect-error mocking the bare minimum of our queries + // get our index version + clients.newClusterClient.asCurrentUser.indices.getMapping.mockResolvedValueOnce({ + body: getIndexMappingsResponseMock('my-index'), + }); + + createSignalsMigrationRoute(server.router); + }); + + test('passes reindex options along to the reindex call', async () => { + const reindexOptions: SignalsReindexOptions = { requests_per_second: 4, size: 10, slices: 2 }; + const request = requestMock.create({ + method: 'post', + path: DETECTION_ENGINE_SIGNALS_MIGRATION_URL, + body: { ...getCreateSignalsMigrationSchemaMock('my-index'), ...reindexOptions }, + }); + + const response = await server.inject(request, context); + + expect(response.status).toEqual(200); + expect(clients.newClusterClient.asCurrentUser.reindex).toHaveBeenCalledWith( + expect.objectContaining({ + body: expect.objectContaining({ + source: { + index: 'my-index', + size: reindexOptions.size, + }, + }), + requests_per_second: reindexOptions.requests_per_second, + slices: reindexOptions.slices, + }) + ); + }); + + it('returns an inline error if write index is out of date but specified', async () => { + clients.appClient.getSignalsIndex.mockReturnValue('my-alias'); + // @ts-expect-error mocking the bare minimum of our queries + // stub index to be write index. + clients.newClusterClient.asCurrentUser.indices.getAlias.mockReset().mockResolvedValueOnce({ + body: { 'my-index': { aliases: { 'my-alias': { is_write_index: true } } } }, + }); + + const request = requestMock.create({ + method: 'post', + path: DETECTION_ENGINE_SIGNALS_MIGRATION_URL, + body: getCreateSignalsMigrationSchemaMock('my-index'), + }); + const response = await server.inject(request, context); + + expect(response.status).toEqual(200); + expect(response.body.indices).toEqual([ + { + error: { + message: 'The specified index is a write index and cannot be migrated.', + status_code: 400, + }, + index: 'my-index', + migration_index: null, + migration_task_id: null, + migration_token: null, + }, + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/create_signals_migration_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/create_signals_migration_route.ts new file mode 100644 index 0000000000000..ca443484bcd9c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/create_signals_migration_route.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'src/core/server'; +import { DETECTION_ENGINE_SIGNALS_MIGRATION_URL } from '../../../../../common/constants'; +import { createSignalsMigrationSchema } from '../../../../../common/detection_engine/schemas/request/create_signals_migration_schema'; +import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; +import { migrateSignals } from '../../migrations/migrate_signals'; +import { buildSiemResponse, transformError } from '../utils'; +import { getTemplateVersion } from '../index/check_template_version'; +import { getMigrationStatus } from '../../migrations/get_migration_status'; +import { encodeMigrationToken, indexIsOutdated } from '../../migrations/helpers'; +import { getIndexAliases } from '../../index/get_index_aliases'; +import { BadRequestError } from '../../errors/bad_request_error'; + +export const createSignalsMigrationRoute = (router: IRouter) => { + router.post( + { + path: DETECTION_ENGINE_SIGNALS_MIGRATION_URL, + validate: { + body: buildRouteValidation(createSignalsMigrationSchema), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + const esClient = context.core.elasticsearch.client.asCurrentUser; + const { index: indices, ...reindexOptions } = request.body; + + try { + const appClient = context.securitySolution?.getAppClient(); + if (!appClient) { + return siemResponse.error({ statusCode: 404 }); + } + + const signalsAlias = appClient.getSignalsIndex(); + const currentVersion = await getTemplateVersion({ + alias: signalsAlias, + esClient, + }); + const signalsIndexAliases = await getIndexAliases({ esClient, alias: signalsAlias }); + + const nonSignalsIndices = indices.filter( + (index) => !signalsIndexAliases.some((alias) => alias.index === index) + ); + if (nonSignalsIndices.length > 0) { + throw new BadRequestError( + `The following indices are not signals indices and cannot be migrated: [${nonSignalsIndices.join()}].` + ); + } + + const migrationStatuses = await getMigrationStatus({ esClient, index: indices }); + const migrationResults = await Promise.all( + indices.map(async (index) => { + const status = migrationStatuses.find(({ name }) => name === index); + if (indexIsOutdated({ status, version: currentVersion })) { + try { + const isWriteIndex = signalsIndexAliases.some( + (alias) => alias.isWriteIndex && alias.index === index + ); + if (isWriteIndex) { + throw new BadRequestError( + 'The specified index is a write index and cannot be migrated.' + ); + } + + const migrationDetails = await migrateSignals({ + esClient, + index, + version: currentVersion, + reindexOptions, + }); + const migrationToken = encodeMigrationToken(migrationDetails); + + return { + index, + migration_index: migrationDetails.destinationIndex, + migration_task_id: migrationDetails.taskId, + migration_token: migrationToken, + }; + } catch (err) { + const error = transformError(err); + return { + index, + error: { + message: error.message, + status_code: error.statusCode, + }, + migration_index: null, + migration_task_id: null, + migration_token: null, + }; + } + } else { + return { + index, + migration_index: null, + migration_task_id: null, + migration_token: null, + }; + } + }) + ); + + return response.ok({ body: { indices: migrationResults } }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.test.ts new file mode 100644 index 0000000000000..207e4c614b9ad --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.test.ts @@ -0,0 +1,49 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { getFinalizeSignalsMigrationRequest } from '../__mocks__/request_responses'; +import { requestContextMock, serverMock } from '../__mocks__'; +import { finalizeSignalsMigrationRoute } from './finalize_signals_migration_route'; + +describe('query for signal', () => { + let server: ReturnType; + let { clients, context } = requestContextMock.createTools(); + + beforeEach(() => { + server = serverMock.create(); + ({ clients, context } = requestContextMock.createTools()); + + // @ts-expect-error mocking the bare minimum of the response + // get our completed task + clients.newClusterClient.asCurrentUser.tasks.get.mockResolvedValueOnce({ + body: { + completed: true, + response: {}, + // satisfies our "is this the right task" validation + task: { description: 'reindexing from sourceIndex to destinationIndex' }, + }, + }); + + // @ts-expect-error mocking the bare minimum of the response + // count of original index + clients.newClusterClient.asCurrentUser.count.mockResolvedValueOnce({ body: { count: 1 } }); + // @ts-expect-error mocking the bare minimum of the response + // count of migrated index + clients.newClusterClient.asCurrentUser.count.mockResolvedValueOnce({ body: { count: 2 } }); + + finalizeSignalsMigrationRoute(server.router); + }); + + test('returns an error if migration index size does not match the original index', async () => { + const response = await server.inject(getFinalizeSignalsMigrationRequest(), context); + expect(response.status).toEqual(500); + expect(response.body).toEqual({ + message: + 'The source and destination indexes have different document counts. Source [sourceIndex] has [1] documents, while destination [destinationIndex] has [2] documents.', + status_code: 500, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.ts new file mode 100644 index 0000000000000..d2f619c218d92 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/finalize_signals_migration_route.ts @@ -0,0 +1,111 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { ReindexResponse } from 'elasticsearch'; + +import { IRouter } from 'src/core/server'; +import { DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL } from '../../../../../common/constants'; +import { finalizeSignalsMigrationSchema } from '../../../../../common/detection_engine/schemas/request/finalize_signals_migration_schema'; +import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; +import { BadRequestError } from '../../errors/bad_request_error'; +import { getIndexCount } from '../../index/get_index_count'; +import { decodeMigrationToken } from '../../migrations/helpers'; +import { applyMigrationCleanupPolicy } from '../../migrations/migration_cleanup'; +import { replaceSignalsIndexAlias } from '../../migrations/replace_signals_index_alias'; +import { buildSiemResponse, transformError } from '../utils'; + +interface TaskResponse { + completed: boolean; + response?: ReindexResponse; + task: { description?: string }; +} + +export const finalizeSignalsMigrationRoute = (router: IRouter) => { + router.post( + { + path: DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL, + validate: { + body: buildRouteValidation(finalizeSignalsMigrationSchema), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + const esClient = context.core.elasticsearch.client.asCurrentUser; + const { migration_token: migrationToken } = request.body; + + try { + const appClient = context.securitySolution?.getAppClient(); + if (!appClient) { + return siemResponse.error({ statusCode: 404 }); + } + + const { destinationIndex, sourceIndex, taskId } = decodeMigrationToken(migrationToken); + const { body: task } = await esClient.tasks.get({ task_id: taskId }); + + if (!task.completed) { + return response.ok({ + body: { + completed: false, + index: sourceIndex, + migration_index: destinationIndex, + migration_task_id: taskId, + migration_token: migrationToken, + }, + }); + } + + const { description } = task.task; + if ( + !description || + !description.includes(destinationIndex) || + !description.includes(sourceIndex) + ) { + throw new BadRequestError( + `The specified task does not match the source and destination indexes. Task [${taskId}] did not specify source index [${sourceIndex}] and destination index [${destinationIndex}]` + ); + } + + const sourceCount = await getIndexCount({ esClient, index: sourceIndex }); + const destinationCount = await getIndexCount({ esClient, index: destinationIndex }); + if (sourceCount !== destinationCount) { + throw new Error( + `The source and destination indexes have different document counts. Source [${sourceIndex}] has [${sourceCount}] documents, while destination [${destinationIndex}] has [${destinationCount}] documents.` + ); + } + + const signalsIndex = appClient.getSignalsIndex(); + await replaceSignalsIndexAlias({ + alias: signalsIndex, + esClient, + newIndex: destinationIndex, + oldIndex: sourceIndex, + }); + + await applyMigrationCleanupPolicy({ alias: signalsIndex, esClient, index: sourceIndex }); + await esClient.delete({ index: '.tasks', id: taskId }); + + return response.ok({ + body: { + completed: true, + index: sourceIndex, + migration_index: destinationIndex, + migration_task_id: taskId, + migration_token: migrationToken, + }, + }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/get_signals_migration_status_route.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/get_signals_migration_status_route.ts new file mode 100644 index 0000000000000..85c38fcdeb01c --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/signals/get_signals_migration_status_route.ts @@ -0,0 +1,66 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { IRouter } from 'src/core/server'; +import { DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL } from '../../../../../common/constants'; +import { getMigrationStatusSchema } from '../../../../../common/detection_engine/schemas/request/get_migration_status_schema'; +import { buildRouteValidation } from '../../../../utils/build_validation/route_validation'; +import { getIndexAliases } from '../../index/get_index_aliases'; +import { getMigrationStatus } from '../../migrations/get_migration_status'; +import { getSignalsIndicesInRange } from '../../migrations/get_signals_indices_in_range'; +import { indexIsOutdated } from '../../migrations/helpers'; +import { getTemplateVersion } from '../index/check_template_version'; +import { buildSiemResponse, transformError } from '../utils'; + +export const getSignalsMigrationStatusRoute = (router: IRouter) => { + router.get( + { + path: DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL, + validate: { + query: buildRouteValidation(getMigrationStatusSchema), + }, + options: { + tags: ['access:securitySolution'], + }, + }, + async (context, request, response) => { + const siemResponse = buildSiemResponse(response); + const esClient = context.core.elasticsearch.client.asCurrentUser; + + try { + const appClient = context.securitySolution?.getAppClient(); + if (!appClient) { + return siemResponse.error({ statusCode: 404 }); + } + + const { from } = request.query; + + const signalsAlias = appClient.getSignalsIndex(); + const currentVersion = await getTemplateVersion({ alias: signalsAlias, esClient }); + const indexAliases = await getIndexAliases({ alias: signalsAlias, esClient }); + const signalsIndices = indexAliases.map((indexAlias) => indexAlias.index); + const indicesInRange = await getSignalsIndicesInRange({ + esClient, + index: signalsIndices, + from, + }); + const migrationStatuses = await getMigrationStatus({ esClient, index: indicesInRange }); + const enrichedStatuses = migrationStatuses.map((status) => ({ + ...status, + is_outdated: indexIsOutdated({ status, version: currentVersion }), + })); + + return response.ok({ body: { indices: enrichedStatuses } }); + } catch (err) { + const error = transformError(err); + return siemResponse.error({ + body: error.message, + statusCode: error.statusCode, + }); + } + } + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts index b613061ac85f2..36131c2e2844d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.test.ts @@ -5,6 +5,7 @@ */ import Boom from '@hapi/boom'; +import { errors } from '@elastic/elasticsearch'; import { SavedObjectsFindResponse } from 'kibana/server'; @@ -97,6 +98,28 @@ describe('utils', () => { statusCode: 400, }); }); + + it('transforms a ResponseError returned by the elasticsearch client', () => { + const error: errors.ResponseError = { + name: 'ResponseError', + message: 'illegal_argument_exception', + headers: {}, + body: { + error: { + type: 'illegal_argument_exception', + reason: 'detailed explanation', + }, + }, + meta: ({} as unknown) as errors.ResponseError['meta'], + statusCode: 400, + }; + const transformed = transformError(error); + + expect(transformed).toEqual({ + message: 'illegal_argument_exception: detailed explanation', + statusCode: 400, + }); + }); }); describe('transformBulkError', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts index 6a304a207d000..e2637ce05b118 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/utils.ts @@ -6,6 +6,7 @@ import Boom from '@hapi/boom'; import Joi from 'joi'; +import { errors } from '@elastic/elasticsearch'; import { has, snakeCase } from 'lodash/fp'; import { SanitizedAlert } from '../../../../../alerts/common'; @@ -24,7 +25,7 @@ export interface OutputError { statusCode: number; } -export const transformError = (err: Error & { statusCode?: number }): OutputError => { +export const transformError = (err: Error & Partial): OutputError => { if (Boom.isBoom(err)) { return { message: err.output.payload.message, @@ -32,10 +33,17 @@ export const transformError = (err: Error & { statusCode?: number }): OutputErro }; } else { if (err.statusCode != null) { - return { - message: err.message, - statusCode: err.statusCode, - }; + if (err.body?.error != null) { + return { + statusCode: err.statusCode, + message: `${err.body.error.type}: ${err.body.error.reason}`, + }; + } else { + return { + statusCode: err.statusCode, + message: err.message, + }; + } } else if (err instanceof BadRequestError) { // allows us to throw request validation errors in the absence of Boom return { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/create_signals_migration.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/create_signals_migration.sh new file mode 100755 index 0000000000000..1beb7cba0289e --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/create_signals_migration.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./signals/create_signals_migration.sh .custom-concrete-signals-index + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/signals/migration \ + -d "{\"index\": [\"$1\"]}" \ + | jq . diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/finalize_signals_migration.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/finalize_signals_migration.sh new file mode 100755 index 0000000000000..1bbc0eef50146 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/finalize_signals_migration.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./signals/finalize_signals_migration.sh eyJkZXN0aW5hdGlvbkluZGV4IjoiZGVzdGluYXRpb25JbmRleCIsInNvdXJjZUluZGV4Ijoic291cmNlSW5kZXgiLCJ0YXNrSWQiOm51bGx9 + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X POST ${KIBANA_URL}${SPACE_URL}/api/detection_engine/signals/finalize_migration \ + -d "{\"migration_token\": \"$1\"}" \ + | jq . diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/get_migration_status.sh b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/get_migration_status.sh new file mode 100755 index 0000000000000..d762fe9212b10 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/scripts/signals/get_migration_status.sh @@ -0,0 +1,18 @@ +#!/bin/sh + +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +set -e +./check_env_variables.sh + +# Example: ./signals/get_migration_status.sh + curl -s -k \ + -H 'Content-Type: application/json' \ + -H 'kbn-xsrf: 123' \ + -u ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD} \ + -X GET ${KIBANA_URL}${SPACE_URL}/api/detection_engine/signals/migration_status?from=now-300d \ + | jq . diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index cd61fbcfd0fc7..f81000091749a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -22,6 +22,7 @@ import { } from './build_bulk_body'; import { SignalHit, SignalSourceHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; +import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; describe('buildBulkBody', () => { beforeEach(() => { @@ -56,6 +57,9 @@ describe('buildBulkBody', () => { kind: 'signal', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, parent: { id: sampleIdGuid, type: 'event', @@ -161,6 +165,9 @@ describe('buildBulkBody', () => { kind: 'signal', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, parent: { id: sampleIdGuid, type: 'event', @@ -269,6 +276,9 @@ describe('buildBulkBody', () => { module: 'system', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, original_event: { action: 'socket_opened', dataset: 'socket', @@ -378,6 +388,9 @@ describe('buildBulkBody', () => { module: 'system', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, original_event: { action: 'socket_opened', dataset: 'socket', @@ -481,6 +494,9 @@ describe('buildBulkBody', () => { kind: 'signal', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, original_event: { kind: 'event', }, @@ -583,6 +599,9 @@ describe('buildBulkBody', () => { kind: 'signal', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, original_signal: 123, parent: { id: sampleIdGuid, @@ -683,6 +702,9 @@ describe('buildBulkBody', () => { kind: 'signal', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, original_signal: { child_1: { child_2: 'nested data' } }, parent: { id: sampleIdGuid, @@ -772,6 +794,9 @@ describe('buildSignalFromSequence', () => { kind: 'signal', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, parents: [ { id: sampleIdGuid, @@ -878,6 +903,9 @@ describe('buildSignalFromSequence', () => { kind: 'signal', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, parents: [ { id: sampleIdGuid, @@ -969,6 +997,9 @@ describe('buildSignalFromEvent', () => { kind: 'signal', }, signal: { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, original_time: '2020-04-20T21:27:45+0000', parent: { id: sampleIdGuid, diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index c5e6bc9f157e0..7823e551193ee 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -18,6 +18,7 @@ import { ANCHOR_DATE, } from '../../../../common/detection_engine/schemas/response/rules_schema.mocks'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; +import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; describe('buildSignal', () => { beforeEach(() => { @@ -33,6 +34,9 @@ describe('buildSignal', () => { ...additionalSignalFields(doc), }; const expected: Signal = { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, parent: { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', @@ -111,6 +115,9 @@ describe('buildSignal', () => { ...additionalSignalFields(doc), }; const expected: Signal = { + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, parent: { id: 'd5e8eb51-a6a0-456d-8a15-4b79bfec3d71', type: 'event', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 9cf2462526cfc..b9a77137396ca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,6 +5,7 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; +import { SIGNALS_TEMPLATE_VERSION } from '../routes/index/get_signals_template'; import { isEventTypeSignal } from './build_event_type_signal'; import { Signal, Ancestor, BaseSignalHit } from './types'; @@ -73,6 +74,9 @@ export const removeClashes = (doc: BaseSignalHit): BaseSignalHit => { * @param rule The rule that is generating the new signal. */ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => { + const _meta = { + version: SIGNALS_TEMPLATE_VERSION, + }; const removedClashes = docs.map(removeClashes); const parents = removedClashes.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; @@ -81,6 +85,7 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => [] ); return { + _meta, parents, ancestors, status: 'open', diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index d6bdc14a92b40..7965a09efefa9 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -67,6 +67,7 @@ import { createThreatSignals } from './threat_mapping/create_threat_signals'; import { getIndexVersion } from '../routes/index/get_index_version'; import { MIN_EQL_RULE_INDEX_VERSION } from '../routes/index/get_signals_template'; import { filterEventsAgainstList } from './filter_events_with_list'; +import { isOutdated } from '../migrations/helpers'; export const signalRulesAlertType = ({ logger, @@ -509,10 +510,7 @@ export const signalRulesAlertType = ({ } try { const signalIndexVersion = await getIndexVersion(services.callCluster, outputIndex); - if ( - signalIndexVersion === undefined || - signalIndexVersion < MIN_EQL_RULE_INDEX_VERSION - ) { + if (isOutdated({ current: signalIndexVersion, target: MIN_EQL_RULE_INDEX_VERSION })) { throw new Error( `EQL based rules require an update to version ${MIN_EQL_RULE_INDEX_VERSION} of the detection alerts index mapping` ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 9e81797b14731..66f3a21dfe680 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -150,6 +150,9 @@ export interface Ancestor { } export interface Signal { + _meta?: { + version: number; + }; rule: RulesSchema; // DEPRECATED: use parents instead of parent parent?: Ancestor; diff --git a/x-pack/plugins/security_solution/server/routes/index.ts b/x-pack/plugins/security_solution/server/routes/index.ts index 3467d0bb66860..1bd92c5c2f079 100644 --- a/x-pack/plugins/security_solution/server/routes/index.ts +++ b/x-pack/plugins/security_solution/server/routes/index.ts @@ -14,8 +14,11 @@ import { findRulesRoute } from '../lib/detection_engine/routes/rules/find_rules_ import { deleteRulesRoute } from '../lib/detection_engine/routes/rules/delete_rules_route'; import { updateRulesRoute } from '../lib/detection_engine/routes/rules/update_rules_route'; import { patchRulesRoute } from '../lib/detection_engine/routes/rules/patch_rules_route'; -import { setSignalsStatusRoute } from '../lib/detection_engine/routes/signals/open_close_signals_route'; +import { createSignalsMigrationRoute } from '../lib/detection_engine/routes/signals/create_signals_migration_route'; +import { finalizeSignalsMigrationRoute } from '../lib/detection_engine/routes/signals/finalize_signals_migration_route'; +import { getSignalsMigrationStatusRoute } from '../lib/detection_engine/routes/signals/get_signals_migration_status_route'; import { querySignalsRoute } from '../lib/detection_engine/routes/signals/query_signals_route'; +import { setSignalsStatusRoute } from '../lib/detection_engine/routes/signals/open_close_signals_route'; import { deleteIndexRoute } from '../lib/detection_engine/routes/index/delete_index_route'; import { readTagsRoute } from '../lib/detection_engine/routes/tags/read_tags_route'; import { readPrivilegesRoute } from '../lib/detection_engine/routes/privileges/read_privileges_route'; @@ -80,6 +83,9 @@ export const initRoutes = ( // Detection Engine Signals routes that have the REST endpoints of /api/detection_engine/signals // POST /api/detection_engine/signals/status // Example usage can be found in security_solution/server/lib/detection_engine/scripts/signals + getSignalsMigrationStatusRoute(router); + createSignalsMigrationRoute(router); + finalizeSignalsMigrationRoute(router); setSignalsStatusRoute(router); querySignalsRoute(router); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts index 9442d911c3fd9..64ee42fdb3f3e 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -25,6 +25,7 @@ import { waitForRuleSuccess, waitForSignalsToBePresent, } from '../../utils'; +import { SIGNALS_TEMPLATE_VERSION } from '../../../../plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template'; /** * Specific _id to use for some of the tests. If the archiver changes and you see errors @@ -123,6 +124,9 @@ export default ({ getService }: FtrProviderContext) => { kind: 'event', module: 'system', }, + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, }); }); @@ -191,6 +195,9 @@ export default ({ getService }: FtrProviderContext) => { kind: 'signal', module: 'system', }, + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, }); }); @@ -244,6 +251,9 @@ export default ({ getService }: FtrProviderContext) => { type: 'event', }, ], + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, }); }); @@ -314,6 +324,9 @@ export default ({ getService }: FtrProviderContext) => { type: 'signal', }, ], + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, }); }); }); @@ -398,6 +411,9 @@ export default ({ getService }: FtrProviderContext) => { }, original_time: '2020-10-28T05:08:53.000Z', original_signal: 1, + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, }); }); @@ -463,6 +479,9 @@ export default ({ getService }: FtrProviderContext) => { original_event: { kind: 'signal', }, + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, }); }); }); @@ -550,6 +569,9 @@ export default ({ getService }: FtrProviderContext) => { }, }, }, + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, }); }); @@ -615,6 +637,9 @@ export default ({ getService }: FtrProviderContext) => { original_event: { kind: 'signal', }, + _meta: { + version: SIGNALS_TEMPLATE_VERSION, + }, }); }); }); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index a2422b9e3bf40..d49d6ed3eedb0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -33,5 +33,6 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./patch_rules')); loadTestFile(require.resolve('./query_signals')); loadTestFile(require.resolve('./open_close_signals')); + loadTestFile(require.resolve('./migrating_signals')); }); }; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/migrating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/migrating_signals.ts new file mode 100644 index 0000000000000..a256b026e5174 --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/migrating_signals.ts @@ -0,0 +1,521 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { + DEFAULT_SIGNALS_INDEX, + DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL, + DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL, + DETECTION_ENGINE_SIGNALS_MIGRATION_URL, +} from '../../../../plugins/security_solution/common/constants'; +import { ROLES } from '../../../../plugins/security_solution/common/test'; +import { encodeMigrationToken } from '../../../../plugins/security_solution/server/lib/detection_engine/migrations/helpers'; +import { SIGNALS_TEMPLATE_VERSION } from '../../../../plugins/security_solution/server/lib/detection_engine/routes/index/get_signals_template'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createSignalsIndex, + deleteSignalsIndex, + getIndexNameFromLoad, + waitFor, + waitForIndexToPopulate, +} from '../../utils'; +import { createUserAndRole } from '../roles_users_utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext): void => { + const es = getService('es'); + const esArchiver = getService('esArchiver'); + const security = getService('security'); + const supertest = getService('supertest'); + const supertestWithoutAuth = getService('supertestWithoutAuth'); + + describe('Migrating signals', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + }); + + describe('migration status of signals indexes', async () => { + let legacySignalsIndexName: string; + + beforeEach(async () => { + legacySignalsIndexName = getIndexNameFromLoad( + await esArchiver.load('signals/legacy_signals_index') + ); + }); + + afterEach(async () => { + await esArchiver.unload('signals/legacy_signals_index'); + }); + + it('returns no indexes if no signals exist in the specified range', async () => { + const { body } = await supertest + .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL) + .query({ from: '2020-10-20' }) + .set('kbn-xsrf', 'true') + .expect(200); + + expect(body.indices).to.eql([]); + }); + + it('includes an index if its signals are within the specified range', async () => { + const { + body: { indices }, + } = await supertest + .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL) + .query({ from: '2020-10-10' }) + .set('kbn-xsrf', 'true') + .expect(200); + + expect(indices).length(1); + expect(indices[0].name).to.eql(legacySignalsIndexName); + }); + + it("returns the mappings version and a breakdown of signals' version", async () => { + const outdatedIndexName = getIndexNameFromLoad( + await esArchiver.load('signals/outdated_signals_index') + ); + + const { body } = await supertest + .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL) + .query({ from: '2020-10-10' }) + .set('kbn-xsrf', 'true') + .expect(200); + + expect(body.indices).to.eql([ + { + name: legacySignalsIndexName, + is_outdated: true, + signal_versions: [ + { + doc_count: 1, + key: 0, + }, + ], + version: 1, + }, + { + is_outdated: true, + name: outdatedIndexName, + signal_versions: [ + { + doc_count: 1, + key: 3, + }, + ], + version: 3, + }, + ]); + + await esArchiver.unload('signals/outdated_signals_index'); + }); + + it('rejects the request if the user does not have sufficient privileges', async () => { + await createUserAndRole(security, ROLES.t1_analyst); + + await supertestWithoutAuth + .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL) + .set('kbn-xsrf', 'true') + .auth(ROLES.t1_analyst, 'changeme') + .query({ from: '2020-10-10' }) + .expect(403); + }); + }); + + describe('Creating a signals migration', async () => { + let legacySignalsIndexName: string; + let outdatedSignalsIndexName: string; + + beforeEach(async () => { + legacySignalsIndexName = getIndexNameFromLoad( + await esArchiver.load('signals/legacy_signals_index') + ); + outdatedSignalsIndexName = getIndexNameFromLoad( + await esArchiver.load('signals/outdated_signals_index') + ); + }); + + afterEach(async () => { + await esArchiver.unload('signals/outdated_signals_index'); + await esArchiver.unload('signals/legacy_signals_index'); + }); + + it('returns the information necessary to finalize the migration', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ index: [legacySignalsIndexName] }) + .expect(200); + + expect(body.indices).length(1); + const [index] = body.indices; + + expect(index.index).to.eql(legacySignalsIndexName); + expect(index.migration_token).to.be.a('string'); + expect(index.migration_token.length).to.be.greaterThan(0); + expect(index.migration_index).not.to.eql(legacySignalsIndexName); + expect(index.migration_index).to.contain(legacySignalsIndexName); + expect(index.migration_task_id).to.be.a('string'); + expect(index.migration_task_id.length).to.be.greaterThan(0); + }); + + it('creates a new index containing migrated signals', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ index: [legacySignalsIndexName, outdatedSignalsIndexName] }) + .expect(200); + + const indices = body.indices as Array<{ migration_token: string; migration_index: string }>; + expect(indices).length(2); + indices.forEach((index) => expect(index.migration_token).to.be.a('string')); + + const [{ migration_index: newIndex }] = indices; + await waitForIndexToPopulate(es, newIndex); + const { body: migrationResults } = await es.search({ index: newIndex }); + + expect(migrationResults.hits.hits).length(1); + const migratedSignal = migrationResults.hits.hits[0]._source.signal; + expect(migratedSignal._meta.version).to.equal(SIGNALS_TEMPLATE_VERSION); + }); + + it('specifying the signals alias itself is a bad request', async () => { + const signalsAlias = `${DEFAULT_SIGNALS_INDEX}-default`; + + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ index: [signalsAlias, legacySignalsIndexName] }) + .expect(400); + + expect(body).to.eql({ + message: + 'The following indices are not signals indices and cannot be migrated: [.siem-signals-default].', + status_code: 400, + }); + }); + + it('rejects extant non-signals indexes', async () => { + const unrelatedIndex = '.tasks'; + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ index: [legacySignalsIndexName, unrelatedIndex] }) + .expect(400); + + expect(body).to.eql({ + message: + 'The following indices are not signals indices and cannot be migrated: [.tasks].', + status_code: 400, + }); + }); + + it('rejects if an unknown index is specified', async () => { + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ index: ['random-index', outdatedSignalsIndexName] }) + .expect(400); + + expect(body).to.eql({ + message: + 'The following indices are not signals indices and cannot be migrated: [random-index].', + status_code: 400, + }); + }); + + it('returns an inline error on a duplicated request as the destination index already exists', async () => { + await supertest + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ index: [legacySignalsIndexName] }) + .expect(200); + + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ index: [legacySignalsIndexName] }) + .expect(200); + + const [{ error, ...info }] = body.indices; + expect(info).to.eql({ + index: legacySignalsIndexName, + migration_index: null, + migration_task_id: null, + migration_token: null, + }); + expect(error.status_code).to.eql(400); + expect(error.message).to.contain('resource_already_exists_exception'); + }); + + it('rejects the request if the user does not have sufficient privileges', async () => { + await createUserAndRole(security, ROLES.t1_analyst); + + await supertestWithoutAuth + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .auth(ROLES.t1_analyst, 'changeme') + .send({ index: [legacySignalsIndexName] }) + .expect(403); + }); + }); + + describe('finalizing signals migrations', async () => { + let legacySignalsIndexName: string; + let outdatedSignalsIndexName: string; + let migratingIndices: any[]; + + beforeEach(async () => { + legacySignalsIndexName = getIndexNameFromLoad( + await esArchiver.load('signals/legacy_signals_index') + ); + outdatedSignalsIndexName = getIndexNameFromLoad( + await esArchiver.load('signals/outdated_signals_index') + ); + + ({ + body: { indices: migratingIndices }, + } = await supertest + .post(DETECTION_ENGINE_SIGNALS_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ index: [legacySignalsIndexName, outdatedSignalsIndexName] }) + .expect(200)); + }); + + afterEach(async () => { + await esArchiver.unload('signals/outdated_signals_index'); + await esArchiver.unload('signals/legacy_signals_index'); + }); + + it('replaces the original index alias with the migrated one', async () => { + const [migratingIndex] = migratingIndices; + + const { body } = await supertest + .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL) + .query({ from: '2020-10-10' }) + .set('kbn-xsrf', 'true') + .expect(200); + const indicesBefore = (body.indices as Array<{ name: string }>).map((index) => index.name); + + expect(indicesBefore).to.contain(migratingIndex.index); + expect(indicesBefore).not.to.contain(migratingIndex.migration_index); + + await waitFor(async () => { + const { + body: { completed }, + } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ migration_token: migratingIndex.migration_token }) + .expect(200); + + return completed; + }, `polling finalize_migration until complete`); + + const { body: bodyAfter } = await supertest + .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL) + .query({ from: '2020-10-10' }) + .set('kbn-xsrf', 'true') + .expect(200); + + const indicesAfter = (bodyAfter.indices as Array<{ name: string }>).map( + (index) => index.name + ); + + expect(indicesAfter).to.contain(migratingIndex.migration_index); + expect(indicesAfter).not.to.contain(migratingIndex.index); + }); + + it('marks the original index for deletion by applying our cleanup policy', async () => { + const [migratingIndex] = migratingIndices; + + await waitFor(async () => { + const { + body: { completed }, + } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ migration_token: migratingIndex.migration_token }) + .expect(200); + + return completed; + }, `polling finalize_migration until complete`); + + const { body } = await es.indices.getSettings({ index: migratingIndex.index }); + const indexSettings = body[migratingIndex.index].settings.index; + expect(indexSettings.lifecycle.name).to.eql( + `${DEFAULT_SIGNALS_INDEX}-default-migration-cleanup` + ); + }); + + it('deletes the original index for deletion by applying our cleanup policy', async () => { + const [migratingIndex] = migratingIndices; + + await waitFor(async () => { + const { + body: { completed }, + } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ migration_token: migratingIndex.migration_token }) + .expect(200); + + return completed; + }, `polling finalize_migration until complete`); + + const { statusCode } = await es.tasks.get( + { task_id: migratingIndex.migration_task_id }, + { ignore: [404] } + ); + expect(statusCode).to.eql(404); + }); + + it('subsequent attempts at finalization are 404s', async () => { + const [migratingIndex] = migratingIndices; + + await waitFor(async () => { + const { + body: { completed }, + } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ migration_token: migratingIndex.migration_token }) + .expect(200); + + return completed; + }, `polling finalize_migration until complete`); + + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ migration_token: migratingIndex.migration_token }) + .expect(404); + + expect(body.status_code).to.eql(404); + expect(body.message).to.contain('resource_not_found_exception'); + + const { body: bodyAfter } = await supertest + .get(DETECTION_ENGINE_SIGNALS_MIGRATION_STATUS_URL) + .query({ from: '2020-10-10' }) + .set('kbn-xsrf', 'true') + .expect(200); + + const indicesAfter = (bodyAfter.indices as Array<{ name: string }>).map( + (index) => index.name + ); + + expect(indicesAfter).to.contain(migratingIndex.migration_index); + expect(indicesAfter).not.to.contain(migratingIndex.index); + }); + + it('rejects if the provided token is invalid', async () => { + const requestBody = { migration_token: 'invalid_token' }; + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send(requestBody) + .expect(400); + + expect(body).to.eql({ + message: 'An error occurred while decoding the migration token: [invalid_token]', + status_code: 400, + }); + }); + + it('rejects if the specified indexes do not match the task', async () => { + const [ + { migration_index: destinationIndex, index: sourceIndex, migration_task_id: taskId }, + ] = migratingIndices; + const migrationDetails = { destinationIndex, sourceIndex, taskId }; + const invalidToken = encodeMigrationToken({ + ...migrationDetails, + sourceIndex: 'bad-index', + }); + const requestBody = { migration_token: invalidToken }; + + let finalizeResponse: any; + + await waitFor(async () => { + const { body, status } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send(requestBody); + finalizeResponse = body; + + return status !== 200; + }, `polling finalize_migration until task is complete (with error)`); + + expect(finalizeResponse).to.eql({ + message: `The specified task does not match the source and destination indexes. Task [${taskId}] did not specify source index [bad-index] and destination index [${destinationIndex}]`, + status_code: 400, + }); + }); + + it('rejects if the task is malformed', async () => { + const [ + { migration_index: destinationIndex, index: sourceIndex, migration_task_id: taskId }, + ] = migratingIndices; + const migrationDetails = { destinationIndex, sourceIndex, taskId }; + const invalidToken = encodeMigrationToken({ + ...migrationDetails, + taskId: 'bad-task-id', + }); + const requestBody = { migration_token: invalidToken }; + + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send(requestBody) + .expect(400); + + expect(body).to.eql({ + message: 'illegal_argument_exception: malformed task id bad-task-id', + status_code: 400, + }); + }); + + it('rejects if the task does not exist', async () => { + const [ + { migration_index: destinationIndex, index: sourceIndex, migration_task_id: taskId }, + ] = migratingIndices; + const migrationDetails = { destinationIndex, sourceIndex, taskId }; + const invalidToken = encodeMigrationToken({ + ...migrationDetails, + taskId: 'oTUltX4IQMOUUVeiohTt8A:124', + }); + const requestBody = { migration_token: invalidToken }; + + const { body } = await supertest + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send(requestBody) + .expect(404); + + expect(body).to.eql({ + message: + "resource_not_found_exception: task [oTUltX4IQMOUUVeiohTt8A:124] belongs to the node [oTUltX4IQMOUUVeiohTt8A] which isn't part of the cluster and there is no record of the task", + status_code: 404, + }); + }); + + it('rejects the request if the user does not have sufficient privileges', async () => { + const [migratingIndex] = migratingIndices; + await createUserAndRole(security, ROLES.t1_analyst); + + await supertestWithoutAuth + .post(DETECTION_ENGINE_SIGNALS_FINALIZE_MIGRATION_URL) + .set('kbn-xsrf', 'true') + .send({ migration_token: migratingIndex.migration_token }) + .auth(ROLES.t1_analyst, 'changeme') + .expect(403); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index c9048eda45fe8..8d8d62cc754a6 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -1040,3 +1040,26 @@ export const createRuleWithExceptionEntries = async ( return ruleResponse; }; + +export const getIndexNameFromLoad = (loadResponse: Record): string => { + const indexNames = Object.keys(loadResponse); + if (indexNames.length > 1) { + throw new Error( + `expected load response to contain one index, but contained multiple: [${indexNames}]` + ); + } + return indexNames[0]; +}; + +/** + * Waits for the given index to contain documents + * + * @param esClient elasticsearch {@link Client} + * @param index name of the index to query + */ +export const waitForIndexToPopulate = async (es: Client, index: string): Promise => { + await waitFor(async () => { + const response = await es.count<{ count: number }>({ index }); + return response.body.count > 0; + }, `waitForIndexToPopulate: ${index}`); +}; diff --git a/x-pack/test/functional/es_archives/signals/README.md b/x-pack/test/functional/es_archives/signals/README.md index 4b147a414f8b3..97c8c504a4039 100644 --- a/x-pack/test/functional/es_archives/signals/README.md +++ b/x-pack/test/functional/es_archives/signals/README.md @@ -1,22 +1,26 @@ Within this folder is input test data for tests such as: ```ts -security_and_spaces/tests/generating_signals.ts +security_and_spaces / tests / generating_signals.ts; ``` where these are small ECS compliant input indexes that try to express tests that exercise different parts of the detection engine signals. Compliant meaning that these might contain extra fields but should not clash with ECS. Nothing stopping anyone from being ECS strict and not having additional extra fields but the extra fields and mappings -are to just try and keep these tests simple and small. Examples are: +are to just try and keep these tests simple and small. Examples include: +#### `signals/numeric_name_clash` -This is an ECS document that has a numeric name clash with a signal structure -``` -numeric_name_clash -``` +An ECS document that has a numeric name clash with a signal structure -This is an ECS document that has an object name clash with a signal structure -``` -object_clash -``` +#### `signals/object_clash` + +An ECS document that has an object name clash with a signal structure + +#### `signals/legacy_signals_index` + +A legacy signals index. It has no migration metadata fields and a very old mapping version. + +#### `signals/outdated_signals_index` +A signals index that had previously been updated but is now out of date. It has migration metadata fields and a recent mapping version. diff --git a/x-pack/test/functional/es_archives/signals/legacy_signals_index/data.json b/x-pack/test/functional/es_archives/signals/legacy_signals_index/data.json new file mode 100644 index 0000000000000..af96194c50556 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/legacy_signals_index/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": ".siem-signals-default-legacy", + "source": { + "@timestamp": "2020-10-10T00:00:00.000Z", + "signal": {} + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/legacy_signals_index/mappings.json b/x-pack/test/functional/es_archives/signals/legacy_signals_index/mappings.json new file mode 100644 index 0000000000000..546e6c273431a --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/legacy_signals_index/mappings.json @@ -0,0 +1,29 @@ +{ + "type": "index", + "value": { + "aliases": { + ".siem-signals-default": { + "is_write_index": false + } + }, + "index": ".siem-signals-default-legacy", + "mappings": { + "_meta": { + "version": 1 + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { "type": "object" } + } + }, + "settings": { + "index": { + "lifecycle": { + "indexing_complete": true + } + } + } + } +} diff --git a/x-pack/test/functional/es_archives/signals/outdated_signals_index/data.json b/x-pack/test/functional/es_archives/signals/outdated_signals_index/data.json new file mode 100644 index 0000000000000..6e401be7ed5dc --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/outdated_signals_index/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": ".siem-signals-default-outdated", + "source": { + "@timestamp": "2020-10-20T00:00:00.000Z", + "signal": { "_meta": { "version": 3 } } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/outdated_signals_index/mappings.json b/x-pack/test/functional/es_archives/signals/outdated_signals_index/mappings.json new file mode 100644 index 0000000000000..6cc0c80288f55 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/outdated_signals_index/mappings.json @@ -0,0 +1,39 @@ +{ + "type": "index", + "value": { + "aliases": { + ".siem-signals-default": { + "is_write_index": false + } + }, + "index": ".siem-signals-default-outdated", + "mappings": { + "_meta": { + "version": 3 + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { + "properties": { + "_meta": { + "properties": { + "version": { + "type": "long" + } + } + } + } + } + } + }, + "settings": { + "index": { + "lifecycle": { + "indexing_complete": true + } + } + } + } +}