diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts index 3a6c9f96b7e881..ca1281e0d2da91 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/aliases.ts @@ -39,7 +39,7 @@ export default ({ getService }: FtrProviderContext) => { await esArchiver.unload('x-pack/test/functional/es_archives/security_solution/alias'); }); - it('Should keep the original alias value such as "host_alias" from a source index when the value is indexed', async () => { + it('should keep the original alias value such as "host_alias" from a source index when the value is indexed', async () => { const rule = getRuleForSignalTesting(['alias']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); @@ -52,7 +52,7 @@ export default ({ getService }: FtrProviderContext) => { }); // TODO: Make aliases work to where we can have ECS fields such as host.name filled out - it.skip('Should copy alias data from a source index into the signals index in the same position when the target is ECS compatible', async () => { + it.skip('should copy alias data from a source index into the signals index in the same position when the target is ECS compatible', async () => { const rule = getRuleForSignalTesting(['alias']); const { id } = await createRule(supertest, rule); await waitForRuleSuccessOrStatus(supertest, id); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 01fa2765ba0f07..fd3675a2e47e62 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -44,6 +44,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./finalize_signals_migrations')); loadTestFile(require.resolve('./delete_signals_migrations')); loadTestFile(require.resolve('./timestamps')); + loadTestFile(require.resolve('./runtime')); }); // That split here enable us on using a different ciGroup to run the tests diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/runtime.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/runtime.ts new file mode 100644 index 00000000000000..4132b63415fccd --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/runtime.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import expect from '@kbn/expect'; + +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createRule, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getRuleForSignalTesting, + getSignalsById, + waitForRuleSuccessOrStatus, + waitForSignalsToBePresent, +} from '../../utils'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + interface HostAlias { + name: string; + hostname: string; + } + + describe('Tests involving runtime fields of source indexes and the signals index', () => { + describe('Regular runtime field mappings', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/runtime'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/runtime'); + }); + + it('should copy normal non-runtime data set from the source index into the signals index in the same position when the target is ECS compatible', async () => { + const rule = getRuleForSignalTesting(['runtime']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((signal) => (signal._source.host as HostAlias).name); + expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']); + }); + + // TODO: Make runtime fields able to be copied to where we can have ECS fields such as host.name filled out by them within the mapping directly + it.skip('should copy "runtime mapping" data from a source index into the signals index in the same position when the target is ECS compatible', async () => { + const rule = getRuleForSignalTesting(['runtime']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map( + (signal) => (signal._source.host_alias as HostAlias).hostname + ); + expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']); + }); + }); + + describe('Runtime field mappings that have conflicts within them', () => { + beforeEach(async () => { + await createSignalsIndex(supertest); + await esArchiver.load('security_solution/runtime_conflicting_fields'); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(supertest); + await esArchiver.unload('security_solution/runtime_conflicting_fields'); + }); + + // TODO: Make the overrides of runtime fields override the host.name in this use case. + it.skip('should copy normal non-runtime data set from the source index into the signals index in the same position when the target is ECS compatible', async () => { + const rule = getRuleForSignalTesting(['runtime_conflicting_fields']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map((signal) => (signal._source.host as HostAlias).name); + expect(hits).to.eql([ + 'I am the [host.name] field value which shadows the original host.name value', + 'I am the [host.name] field value which shadows the original host.name value', + 'I am the [host.name] field value which shadows the original host.name value', + 'I am the [host.name] field value which shadows the original host.name value', + ]); + }); + + // TODO: Make runtime fields able to be copied to where we can have ECS fields such as host.name filled out by them within the mapping directly + it.skip('should copy "runtime mapping" data from a source index into the signals index in the same position when the target is ECS compatible', async () => { + const rule = getRuleForSignalTesting(['runtime_conflicting_fields']); + const { id } = await createRule(supertest, rule); + await waitForRuleSuccessOrStatus(supertest, id); + await waitForSignalsToBePresent(supertest, 4, [id]); + const signalsOpen = await getSignalsById(supertest, id); + const hits = signalsOpen.hits.hits.map( + (signal) => (signal._source.host_alias as HostAlias).hostname + ); + expect(hits).to.eql(['host name 1', 'host name 2', 'host name 3', 'host name 4']); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 7456040a436a4b..54252b19fc940f 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -1007,7 +1007,7 @@ export const waitForRuleSuccessOrStatus = async ( .send({ ids: [id] }) .expect(200); return body[id]?.current_status?.status === status; - }, 'waitForRuleSuccess'); + }, 'waitForRuleSuccessOrStatus'); }; /** diff --git a/x-pack/test/functional/es_archives/security_solution/runtime/data.json b/x-pack/test/functional/es_archives/security_solution/runtime/data.json new file mode 100644 index 00000000000000..799d59e6fb1eec --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/runtime/data.json @@ -0,0 +1,59 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "runtime", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "host": { + "name": "host name 1" + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "runtime", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "host": { + "name": "host name 2" + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "runtime", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "host": { + "name": "host name 3" + } + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "runtime", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "host": { + "name": "host name 4" + } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/runtime/mappings.json b/x-pack/test/functional/es_archives/security_solution/runtime/mappings.json new file mode 100644 index 00000000000000..33235d0171837c --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/runtime/mappings.json @@ -0,0 +1,36 @@ +{ + "type": "index", + "value": { + "index": "runtime", + "mappings": { + "dynamic": "strict", + "runtime": { + "host.hostname": { + "type": "keyword", + "script": { + "source": "emit(doc['host.name'].value)" + } + } + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "name": { + "type": "keyword" + } + } + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/data.json b/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/data.json new file mode 100644 index 00000000000000..ea64cd215f478c --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/data.json @@ -0,0 +1,67 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "runtime_conflicting_fields", + "source": { + "@timestamp": "2020-10-28T05:00:53.000Z", + "host": [{ + "name": "host name 1_1" + }, { + "name": "host name 1_2" + }] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "2", + "index": "runtime_conflicting_fields", + "source": { + "@timestamp": "2020-10-28T05:01:53.000Z", + "host": [{ + "name": "host name 2_1" + }, { + "name": "host name 2_2" + }] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "3", + "index": "runtime_conflicting_fields", + "source": { + "@timestamp": "2020-10-28T05:02:53.000Z", + "host": [{ + "name": "host name 3_1" + }, { + "name": "host name 3_2" + }] + }, + "type": "_doc" + } +} + +{ + "type": "doc", + "value": { + "id": "4", + "index": "runtime_conflicting_fields", + "source": { + "@timestamp": "2020-10-28T05:03:53.000Z", + "host": [{ + "name": "host name 4_1" + }, { + "name": "host name 4_2" + }] + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/mappings.json b/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/mappings.json new file mode 100644 index 00000000000000..04a538a3329532 --- /dev/null +++ b/x-pack/test/functional/es_archives/security_solution/runtime_conflicting_fields/mappings.json @@ -0,0 +1,114 @@ +{ + "type": "index", + "value": { + "index": "runtime_conflicting_fields", + "mappings": { + "dynamic": "strict", + "runtime": { + "host_alias": { + "type": "keyword", + "script": { + "source": "emit(doc['host.name'].value)" + } + }, + ".": { + "type": "keyword", + "script": { + "source": "emit('I am the [.] field that runtime exposes')" + } + }, + "..": { + "type": "keyword", + "script": { + "source": "emit('I am the [..] field that runtime exposes')" + } + }, + "host.name.": { + "type": "keyword", + "script": { + "source": "emit('I am the [host.name] field that runtime exposes')" + } + }, + ".host.name": { + "type": "keyword", + "script": { + "source": "emit('I am the [.host.name] field that runtime exposes')" + } + }, + ".host.name.": { + "type": "keyword", + "script": { + "source": "emit('I am the [.host.name.] field that runtime exposes')" + } + }, + "..host.name.": { + "type": "keyword", + "script": { + "source": "emit('I am the [..host.name.] field that runtime exposes')" + } + }, + "..host.name": { + "type": "keyword", + "script": { + "source": "emit('I am the [..host.name] field that runtime exposes')" + } + }, + ".host.name..": { + "type": "keyword", + "script": { + "source": "emit('I am the [.host.name..] field that runtime exposes')" + } + }, + "host.name..": { + "type": "keyword", + "script": { + "source": "emit('I am the [host.name..] field that runtime exposes')" + } + }, + "host": { + "type": "keyword", + "script": { + "source": "emit('I am the [host] field which is now a keyword instead of object runtime exposes')" + } + }, + "host.name": { + "type": "keyword", + "script": { + "source": "emit('I am the [host.name] field value which shadows the original host.name value')" + } + }, + "host.field_1": { + "type": "keyword", + "script": { + "source": "emit('I am the [host.name.field_1] field value which now confuses which array element I should be in and am ambiguous')" + } + }, + "host.name.subobject": { + "type": "keyword", + "script": { + "source": "emit('I am the [host.name.subobject] field value which is now an object instead of keyword runtime exposes')" + } + } + }, + "properties": { + "@timestamp": { + "type": "date" + }, + "host": { + "properties": { + "name": { + "type": "keyword" + } + } + } + } + }, + "settings": { + "index": { + "refresh_interval": "1s", + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}