From b6d661f9c38f1c83a747e98b99594ecc8b567b76 Mon Sep 17 00:00:00 2001 From: Frank Hassanabad Date: Fri, 6 Nov 2020 07:47:57 -0700 Subject: [PATCH] [Security Solutions][Detection Engine] Fixes critical clashing with source indexes that already contain a "signal" field (#82191) ## Summary Fixes: https://github.com/elastic/kibana/issues/82148 We have errors and do not generate a signal when a source index already has utilized and reserved the "signal" field for their own data purposes. This fix is a bit tricky and has one medium sized risk which is we also support "signals generated on top of existing signals". Therefore we have to be careful and do a small runtime detection of the "data shape" of the signal's data type. If it looks like the user is using the "signal" field within their mapping instead of us, we move the customer's signal into "original_signal" inside our "signal" structure we create when we copy their data set when creating a signal. To help mitigate the risks associated with this critical bug with regards to breaking signals on top of signals I have: * This adds unit tests * This adds end to end tests for testing generating signals including signals on signals to help mitigate risk The key test for this shape in the PR are in the file: ``` detection_engine/signals/build_event_type_signal.ts ``` like so: ```ts export const isEventTypeSignal = (doc: BaseSignalHit): boolean => { return doc._source.signal?.rule?.id != null && typeof doc._source.signal?.rule?.id === 'string'; }; ``` Example of what happens when it does a "move" of an existing numeric signal keyword type: ```ts # This causes a clash with us using the name signal as a numeric. PUT clashing-index/_doc/1 { "@timestamp": "2020-10-28T05:08:53.000Z", "signal": 1 } ``` Before, this was an error. With this PR it now will restructure this data like so when creating a signal along with additional signal ancestor information, meta data. I omitted some of the data from the output signal for this example. ```ts { ... Other data copied ... "signal": { "original_signal": 1 <--- We "move it" here now "parents": [ { "id": "BhbXBmkBR346wHgn4PeZ", "type": "event", "index": "your-index-name", "depth": 0 }, ], "ancestors": [ { "id": "BhbXBmkBR346wHgn4PeZ", "type": "event", "index": "your-index-name", "depth": 0 }, ], "status": "open", "depth": 1, "parent": { "id": "BhbXBmkBR346wHgn4PeZ", type: "event", "index": "your-index-name", "depth": 0 }, "original_time": "2019-02-19T17:40:03.790Z", "original_event": { "action": "socket_closed", "dataset": "socket", "kind": "event", "module": "system" }, } ``` ### Checklist - [x] [Unit or functional tests](https://www.elastic.co/guide/en/kibana/master/development-tests.html) were updated or added to match the most common scenarios --- .../routes/index/signals_mapping.json | 5 + .../signals/build_bulk_body.test.ts | 202 ++++++- .../signals/build_bulk_body.ts | 2 +- .../signals/build_event_type_signal.test.ts | 56 +- .../signals/build_event_type_signal.ts | 13 + .../signals/build_signal.test.ts | 70 ++- .../detection_engine/signals/build_signal.ts | 31 +- .../signals/single_bulk_create.test.ts | 12 + .../signals/single_bulk_create.ts | 3 +- .../lib/detection_engine/signals/types.ts | 1 + .../tests/generating_signals.ts | 508 ++++++++++++++++++ .../security_and_spaces/tests/index.ts | 1 + .../detection_engine_api_integration/utils.ts | 29 + .../signals/numeric_name_clash/data.json | 12 + .../signals/numeric_name_clash/mappings.json | 20 + .../signals/object_clash/data.json | 12 + .../signals/object_clash/mappings.json | 20 + 17 files changed, 988 insertions(+), 9 deletions(-) create mode 100644 x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts create mode 100644 x-pack/test/functional/es_archives/signals/numeric_name_clash/data.json create mode 100644 x-pack/test/functional/es_archives/signals/numeric_name_clash/mappings.json create mode 100644 x-pack/test/functional/es_archives/signals/object_clash/data.json create mode 100644 x-pack/test/functional/es_archives/signals/object_clash/mappings.json diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 7255325358baf..4e9477f3f2f73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -257,6 +257,11 @@ "original_time": { "type": "date" }, + "original_signal": { + "type": "object", + "dynamic": false, + "enabled": false + }, "original_event": { "properties": { "action": { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 75a7de8cd2c44..ad060a9304e84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -20,7 +20,7 @@ import { objectPairIntersection, objectArrayIntersection, } from './build_bulk_body'; -import { SignalHit } from './types'; +import { SignalHit, SignalSourceHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; describe('buildBulkBody', () => { @@ -441,6 +441,206 @@ describe('buildBulkBody', () => { }; expect(fakeSignalSourceHit).toEqual(expected); }); + + test('bulk body builds "original_signal" if it exists already as a numeric', () => { + const sampleParams = sampleRuleAlertParams(); + const sampleDoc = sampleDocNoSortId(); + delete sampleDoc._source.source; + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: 123, + }, + } as unknown) as SignalSourceHit; + const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + const expected: Omit & { someKey: string } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_signal: 123, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); + + test('bulk body builds "original_signal" if it exists already as an object', () => { + const sampleParams = sampleRuleAlertParams(); + const sampleDoc = sampleDocNoSortId(); + delete sampleDoc._source.source; + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { child_1: { child_2: 'nested data' } }, + }, + } as unknown) as SignalSourceHit; + const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + const expected: Omit & { someKey: string } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_signal: { child_1: { child_2: 'nested data' } }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); }); describe('buildSignalFromSequence', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index cc454ac1e9462..a704d076880bf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -154,7 +154,7 @@ export const buildSignalFromEvent = ( const rule = applyOverrides ? buildRuleWithOverrides(ruleSO, event._source) : buildRuleWithoutOverrides(ruleSO); - const signal = { + const signal: Signal = { ...buildSignal([event], rule), ...additionalSignalFields(event), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts index 106a049002e05..ada939ed0941a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts @@ -5,7 +5,8 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { buildEventTypeSignal } from './build_event_type_signal'; +import { buildEventTypeSignal, isEventTypeSignal } from './build_event_type_signal'; +import { BaseSignalHit } from './types'; describe('buildEventTypeSignal', () => { beforeEach(() => { @@ -44,4 +45,57 @@ describe('buildEventTypeSignal', () => { }; expect(eventType).toEqual(expected); }); + + test('It validates a sample doc with no signal type as "false"', () => { + const doc = sampleDocNoSortId(); + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates a sample doc with a signal type as "true"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: { + rule: { id: 'id-123' }, + }, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(true); + }); + + test('It validates a numeric signal string as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: 'something', + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates an empty object as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: {}, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates an empty rule object as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: { + rule: {}, + }, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts index 81c9d1dedcc56..3d78cf5ce5e46 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts @@ -13,3 +13,16 @@ export const buildEventTypeSignal = (doc: BaseSignalHit): object => { return { kind: 'signal' }; } }; + +/** + * Given a document this will return true if that document is a signal + * document. We can't guarantee the code will call this function with a document + * before adding the _source.event.kind = "signal" from "buildEventTypeSignal" + * so we do basic testing to ensure that if the object has the fields of: + * "signal.rule.id" then it will be one of our signals rather than a customer + * overwritten signal. + * @param doc The document which might be a signal or it might be a regular log + */ +export const isEventTypeSignal = (doc: BaseSignalHit): boolean => { + return doc._source.signal?.rule?.id != null && typeof doc._source.signal?.rule?.id === 'string'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index d0c451bbdf2e2..c5e6bc9f157e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -5,8 +5,14 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; -import { Signal, Ancestor } from './types'; +import { + buildSignal, + buildParent, + buildAncestors, + additionalSignalFields, + removeClashes, +} from './build_signal'; +import { Signal, Ancestor, BaseSignalHit } from './types'; import { getRulesSchemaMock, ANCHOR_DATE, @@ -302,4 +308,64 @@ describe('buildSignal', () => { ]; expect(signal).toEqual(expected); }); + + describe('removeClashes', () => { + test('it will call renameClashes with a regular doc and not mutate it if it does not have a signal clash', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const output = removeClashes(doc); + expect(output).toBe(doc); // reference check + }); + + test('it will call renameClashes with a regular doc and not change anything', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const output = removeClashes(doc); + expect(output).toEqual(doc); // deep equal check + }); + + test('it will remove a "signal" numeric clash', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: 127, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')); + }); + + test('it will remove a "signal" object clash', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { child_1: { child_2: 'Test nesting' } }, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')); + }); + + test('it will not remove a "signal" if that is signal is one of our signals', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { rule: { id: '123' } }, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + const expected = { + ...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'), + _source: { + ...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')._source, + signal: { rule: { id: '123' } }, + }, + }; + expect(output).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 947938de6caca..b36a1cbb4a6b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,6 +5,7 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; +import { isEventTypeSignal } from './build_event_type_signal'; import { Signal, Ancestor, BaseSignalHit } from './types'; /** @@ -48,15 +49,37 @@ export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => { } }; +/** + * This removes any signal named clashes such as if a source index has + * "signal" but is not a signal object we put onto the object. If this + * is our "signal object" then we don't want to remove it. + * @param doc The source index doc to a signal. + */ +export const removeClashes = (doc: BaseSignalHit): BaseSignalHit => { + const { signal, ...noSignal } = doc._source; + if (signal == null || isEventTypeSignal(doc)) { + return doc; + } else { + return { + ...doc, + _source: { ...noSignal }, + }; + } +}; + /** * Builds the `signal.*` fields that are common across all signals. * @param docs The parent signals/events of the new signal to be built. * @param rule The rule that is generating the new signal. */ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => { - const parents = docs.map(buildParent); + const removedClashes = docs.map(removeClashes); + const parents = removedClashes.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; - const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); + const ancestors = removedClashes.reduce( + (acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), + [] + ); return { parents, ancestors, @@ -72,9 +95,11 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => */ export const additionalSignalFields = (doc: BaseSignalHit) => { return { - parent: buildParent(doc), + parent: buildParent(removeClashes(doc)), original_time: doc._source['@timestamp'], original_event: doc._source.event ?? undefined, threshold_count: doc._source.threshold_count ?? undefined, + original_signal: + doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts index b7cc13fd13a01..eeeda6561892d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -330,6 +330,18 @@ describe('singleBulkCreate', () => { ]); }); + test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own numeric signal type', () => { + const doc = { ...sampleDocWithAncestors(), _source: { signal: 1234 } }; + const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); + expect(filtered).toEqual([]); + }); + + test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own object signal type', () => { + const doc = { ...sampleDocWithAncestors(), _source: { signal: {} } }; + const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); + expect(filtered).toEqual([]); + }); + test('create successful and returns proper createdItemsCount', async () => { const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 759890cc9d074..d8889dcfcf471 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -14,6 +14,7 @@ import { generateId, makeFloatString, errorAggregator } from './utils'; import { buildBulkBody } from './build_bulk_body'; import { BuildRuleMessage } from './rule_messages'; import { Logger } from '../../../../../../../src/core/server'; +import { isEventTypeSignal } from './build_event_type_signal'; interface SingleBulkCreateParams { filteredEvents: SignalSearchResponse; @@ -50,7 +51,7 @@ export const filterDuplicateRules = ( signalSearchResponse: SignalSearchResponse ) => { return signalSearchResponse.hits.hits.filter((doc) => { - if (doc._source.signal == null) { + if (doc._source.signal == null || !isEventTypeSignal(doc)) { return true; } else { return !( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 9d4e7d8a81051..7128feb80ab3c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -157,6 +157,7 @@ export interface Signal { original_event?: SearchTypes; status: Status; threshold_count?: SearchTypes; + original_signal?: SearchTypes; depth: number; } diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts new file mode 100644 index 0000000000000..0cd1c21447dfe --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -0,0 +1,508 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { CreateRulesSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request'; +import { DEFAULT_SIGNALS_INDEX } from '../../../../plugins/security_solution/common/constants'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createRule, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getAllSignals, + getSignalsByRuleIds, + getSimpleRule, + waitForSignalsToBePresent, +} from '../../utils'; + +/** + * Specific _id to use for some of the tests. If the archiver changes and you see errors + * here, update this to a new value of a chosen auditbeat record and update the tests values. + */ +export const ID = 'BhbXBmkBR346wHgn4PeZ'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + describe('Generating signals from source indexes', () => { + beforeEach(async () => { + await deleteAllAlerts(es); + await createSignalsIndex(supertest); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(es); + }); + + describe('Signals from audit beat are of the expected structure', () => { + beforeEach(async () => { + await esArchiver.load('auditbeat/hosts'); + }); + + afterEach(async () => { + await esArchiver.unload('auditbeat/hosts'); + }); + + it('should have the specific audit record for _id or none of these tests below will pass', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + from: '1900-01-01T00:00:00.000Z', + query: `_id:${ID}`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits.length).greaterThan(0); + }); + + it('should have recorded the rule_id within the signal', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + from: '1900-01-01T00:00:00.000Z', + query: `_id:${ID}`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits[0]._source.signal.rule.rule_id).eql(getSimpleRule().rule_id); + }); + + it('should query and get back expected signal structure using a basic KQL query', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + from: '1900-01-01T00:00:00.000Z', + query: `_id:${ID}`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + expect(signalNoRule).eql({ + parents: [ + { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + ], + ancestors: [ + { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + ], + status: 'open', + depth: 1, + parent: { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + original_time: '2019-02-19T17:40:03.790Z', + original_event: { + action: 'socket_closed', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + }); + }); + + it('should query and get back expected signal structure when it is a signal on a signal', async () => { + // create a 1 signal from 1 auditbeat record + const rule: CreateRulesSchema = { + ...getSimpleRule(), + from: '1900-01-01T00:00:00.000Z', + query: `_id:${ID}`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + + // Run signals on top of that 1 signal which should create a single signal (on top of) a signal + const ruleForSignals: CreateRulesSchema = { + ...getSimpleRule(), + rule_id: 'signal-on-signal', + index: [`${DEFAULT_SIGNALS_INDEX}*`], + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + }; + await createRule(supertest, ruleForSignals); + await waitForSignalsToBePresent(supertest, 2); + + // Get our single signal on top of a signal + const signalsOpen = await getSignalsByRuleIds(supertest, ['signal-on-signal']); + + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + expect(signalNoRule).eql({ + parents: [ + { + rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it + id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + ancestors: [ + { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + { + rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it + id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + status: 'open', + depth: 2, + parent: { + rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it + id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + original_time: signalNoRule.original_time, // original_time will always be changing sine it's based on a signal created here, so skip testing it + original_event: { + action: 'socket_closed', + dataset: 'socket', + kind: 'signal', + module: 'system', + }, + }); + }); + }); + + /** + * These are a set of tests for whenever someone sets up their source + * index to have a name and mapping clash against "signal" with a numeric value. + * You should see the "signal" name/clash being copied to "original_signal" + * underneath the signal object and no errors when they do have a clash. + */ + describe('Signals generated from name clashes', () => { + beforeEach(async () => { + await esArchiver.load('signals/numeric_name_clash'); + }); + + afterEach(async () => { + await esArchiver.unload('signals/numeric_name_clash'); + }); + + it('should have the specific audit record for _id or none of these tests below will pass', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_name_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits.length).greaterThan(0); + }); + + it('should have recorded the rule_id within the signal', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_name_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits[0]._source.signal.rule.rule_id).eql(getSimpleRule().rule_id); + }); + + it('should query and get back expected signal structure using a basic KQL query', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_name_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + expect(signalNoRule).eql({ + parents: [ + { + id: '1', + type: 'event', + index: 'signal_name_clash', + depth: 0, + }, + ], + ancestors: [ + { + id: '1', + type: 'event', + index: 'signal_name_clash', + depth: 0, + }, + ], + status: 'open', + depth: 1, + parent: { + id: '1', + type: 'event', + index: 'signal_name_clash', + depth: 0, + }, + original_time: '2020-10-28T05:08:53.000Z', + original_signal: 1, + }); + }); + + it('should query and get back expected signal structure when it is a signal on a signal', async () => { + // create a 1 signal from 1 auditbeat record + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_name_clash'], + from: '1900-01-01T00:00:00.000Z', + query: `_id:1`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + + // Run signals on top of that 1 signal which should create a single signal (on top of) a signal + const ruleForSignals: CreateRulesSchema = { + ...getSimpleRule(), + rule_id: 'signal-on-signal', + index: [`${DEFAULT_SIGNALS_INDEX}*`], + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + }; + await createRule(supertest, ruleForSignals); + await waitForSignalsToBePresent(supertest, 2); + + // Get our single signal on top of a signal + const signalsOpen = await getSignalsByRuleIds(supertest, ['signal-on-signal']); + + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + + expect(signalNoRule).eql({ + parents: [ + { + rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it + id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + ancestors: [ + { + id: '1', + type: 'event', + index: 'signal_name_clash', + depth: 0, + }, + { + rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it + id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + status: 'open', + depth: 2, + parent: { + rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it + id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + original_time: signalNoRule.original_time, // original_time will always be changing sine it's based on a signal created here, so skip testing it + original_event: { + kind: 'signal', + }, + }); + }); + }); + + /** + * These are a set of tests for whenever someone sets up their source + * index to have a name and mapping clash against "signal" with an object value. + * You should see the "signal" object/clash being copied to "original_signal" underneath + * the signal object and no errors when they do have a clash. + */ + describe('Signals generated from name clashes', () => { + beforeEach(async () => { + await esArchiver.load('signals/object_clash'); + }); + + afterEach(async () => { + await esArchiver.unload('signals/object_clash'); + }); + + it('should have the specific audit record for _id or none of these tests below will pass', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_object_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits.length).greaterThan(0); + }); + + it('should have recorded the rule_id within the signal', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_object_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits[0]._source.signal.rule.rule_id).eql(getSimpleRule().rule_id); + }); + + it('should query and get back expected signal structure using a basic KQL query', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_object_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + expect(signalNoRule).eql({ + parents: [ + { + id: '1', + type: 'event', + index: 'signal_object_clash', + depth: 0, + }, + ], + ancestors: [ + { + id: '1', + type: 'event', + index: 'signal_object_clash', + depth: 0, + }, + ], + status: 'open', + depth: 1, + parent: { + id: '1', + type: 'event', + index: 'signal_object_clash', + depth: 0, + }, + original_time: '2020-10-28T05:08:53.000Z', + original_signal: { + child_1: { + child_2: { + value: 'some_value', + }, + }, + }, + }); + }); + + it('should query and get back expected signal structure when it is a signal on a signal', async () => { + // create a 1 signal from 1 auditbeat record + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_object_clash'], + from: '1900-01-01T00:00:00.000Z', + query: `_id:1`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + + // Run signals on top of that 1 signal which should create a single signal (on top of) a signal + const ruleForSignals: CreateRulesSchema = { + ...getSimpleRule(), + rule_id: 'signal-on-signal', + index: [`${DEFAULT_SIGNALS_INDEX}*`], + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + }; + await createRule(supertest, ruleForSignals); + await waitForSignalsToBePresent(supertest, 2); + + // Get our single signal on top of a signal + const signalsOpen = await getSignalsByRuleIds(supertest, ['signal-on-signal']); + + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + + expect(signalNoRule).eql({ + parents: [ + { + rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it + id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + ancestors: [ + { + id: '1', + type: 'event', + index: 'signal_object_clash', + depth: 0, + }, + { + rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it + id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + status: 'open', + depth: 2, + parent: { + rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it + id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + original_time: signalNoRule.original_time, // original_time will always be changing sine it's based on a signal created here, so skip testing it + original_event: { + kind: 'signal', + }, + }); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 24b76853164f2..962ae53b1241f 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -22,6 +22,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./export_rules')); loadTestFile(require.resolve('./find_rules')); loadTestFile(require.resolve('./find_statuses')); + loadTestFile(require.resolve('./generating_signals')); loadTestFile(require.resolve('./get_prepackaged_rules_status')); loadTestFile(require.resolve('./import_rules')); loadTestFile(require.resolve('./read_rules')); diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 05a0f73dd0dc4..c5e417c710283 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -143,6 +143,19 @@ export const getQuerySignalIds = (signalIds: SignalIds) => ({ }, }); +/** + * Given an array of ruleIds for a test this will get the signals + * created from that rule_id. + * @param ruleIds The rule_id to search for signals + */ +export const getQuerySignalsRuleId = (ruleIds: string[]) => ({ + query: { + terms: { + 'signal.rule.rule_id': ruleIds, + }, + }, +}); + export const setSignalStatus = ({ signalIds, status, @@ -834,6 +847,22 @@ export const getAllSignals = async ( return signalsOpen; }; +export const getSignalsByRuleIds = async ( + supertest: SuperTest, + ruleIds: string[] +): Promise< + SearchResponse<{ + signal: Signal; + }> +> => { + const { body: signalsOpen }: { body: SearchResponse<{ signal: Signal }> } = await supertest + .post(DETECTION_ENGINE_QUERY_SIGNALS_URL) + .set('kbn-xsrf', 'true') + .send(getQuerySignalsRuleId(ruleIds)) + .expect(200); + return signalsOpen; +}; + export const installPrePackagedRules = async ( supertest: SuperTest ): Promise => { diff --git a/x-pack/test/functional/es_archives/signals/numeric_name_clash/data.json b/x-pack/test/functional/es_archives/signals/numeric_name_clash/data.json new file mode 100644 index 0000000000000..ca2cf0de2d845 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/numeric_name_clash/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "signal_name_clash", + "source": { + "@timestamp": "2020-10-28T05:08:53.000Z", + "signal": 1 + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/numeric_name_clash/mappings.json b/x-pack/test/functional/es_archives/signals/numeric_name_clash/mappings.json new file mode 100644 index 0000000000000..98823cb4b4250 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/numeric_name_clash/mappings.json @@ -0,0 +1,20 @@ +{ + "type": "index", + "value": { + "index": "signal_name_clash", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { "type": "keyword" } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/signals/object_clash/data.json b/x-pack/test/functional/es_archives/signals/object_clash/data.json new file mode 100644 index 0000000000000..d2f844312e8fb --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/object_clash/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "signal_object_clash", + "source": { + "@timestamp": "2020-10-28T05:08:53.000Z", + "signal": { "child_1": { "child_2": { "value": "some_value" } } } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/object_clash/mappings.json b/x-pack/test/functional/es_archives/signals/object_clash/mappings.json new file mode 100644 index 0000000000000..9297ff3e867c9 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/object_clash/mappings.json @@ -0,0 +1,20 @@ +{ + "type": "index", + "value": { + "index": "signal_object_clash", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { "type": "object" } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}