diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json index 7255325358baf..4e9477f3f2f73 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/routes/index/signals_mapping.json @@ -257,6 +257,11 @@ "original_time": { "type": "date" }, + "original_signal": { + "type": "object", + "dynamic": false, + "enabled": false + }, "original_event": { "properties": { "action": { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts index 75a7de8cd2c44..ad060a9304e84 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.test.ts @@ -20,7 +20,7 @@ import { objectPairIntersection, objectArrayIntersection, } from './build_bulk_body'; -import { SignalHit } from './types'; +import { SignalHit, SignalSourceHit } from './types'; import { getListArrayMock } from '../../../../common/detection_engine/schemas/types/lists.mock'; describe('buildBulkBody', () => { @@ -441,6 +441,206 @@ describe('buildBulkBody', () => { }; expect(fakeSignalSourceHit).toEqual(expected); }); + + test('bulk body builds "original_signal" if it exists already as a numeric', () => { + const sampleParams = sampleRuleAlertParams(); + const sampleDoc = sampleDocNoSortId(); + delete sampleDoc._source.source; + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: 123, + }, + } as unknown) as SignalSourceHit; + const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + const expected: Omit & { someKey: string } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_signal: 123, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); + + test('bulk body builds "original_signal" if it exists already as an object', () => { + const sampleParams = sampleRuleAlertParams(); + const sampleDoc = sampleDocNoSortId(); + delete sampleDoc._source.source; + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { child_1: { child_2: 'nested data' } }, + }, + } as unknown) as SignalSourceHit; + const { '@timestamp': timestamp, ...fakeSignalSourceHit } = buildBulkBody({ + doc, + ruleParams: sampleParams, + id: sampleRuleGuid, + name: 'rule-name', + actions: [], + createdAt: '2020-01-28T15:58:34.810Z', + updatedAt: '2020-01-28T15:59:14.004Z', + createdBy: 'elastic', + updatedBy: 'elastic', + interval: '5m', + enabled: true, + tags: ['some fake tag 1', 'some fake tag 2'], + throttle: 'no_actions', + }); + const expected: Omit & { someKey: string } = { + someKey: 'someValue', + event: { + kind: 'signal', + }, + signal: { + original_signal: { child_1: { child_2: 'nested data' } }, + parent: { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + parents: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + ancestors: [ + { + id: sampleIdGuid, + type: 'event', + index: 'myFakeSignalIndex', + depth: 0, + }, + ], + original_time: '2020-04-20T21:27:45+0000', + status: 'open', + rule: { + actions: [], + author: ['Elastic'], + building_block_type: 'default', + id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', + rule_id: 'rule-1', + false_positives: [], + max_signals: 10000, + risk_score: 50, + risk_score_mapping: [], + output_index: '.siem-signals', + description: 'Detecting root and admin users', + from: 'now-6m', + immutable: false, + index: ['auditbeat-*', 'filebeat-*', 'packetbeat-*', 'winlogbeat-*'], + interval: '5m', + language: 'kuery', + license: 'Elastic License', + name: 'rule-name', + query: 'user.name: root or user.name: admin', + references: ['http://google.com'], + severity: 'high', + severity_mapping: [], + tags: ['some fake tag 1', 'some fake tag 2'], + threat: [], + type: 'query', + to: 'now', + note: '', + enabled: true, + created_by: 'elastic', + updated_by: 'elastic', + version: 1, + updated_at: fakeSignalSourceHit.signal.rule?.updated_at, + created_at: fakeSignalSourceHit.signal.rule?.created_at, + throttle: 'no_actions', + exceptions_list: getListArrayMock(), + }, + depth: 1, + }, + }; + expect(fakeSignalSourceHit).toEqual(expected); + }); }); describe('buildSignalFromSequence', () => { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts index cc454ac1e9462..a704d076880bf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_bulk_body.ts @@ -154,7 +154,7 @@ export const buildSignalFromEvent = ( const rule = applyOverrides ? buildRuleWithOverrides(ruleSO, event._source) : buildRuleWithoutOverrides(ruleSO); - const signal = { + const signal: Signal = { ...buildSignal([event], rule), ...additionalSignalFields(event), }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts index 106a049002e05..ada939ed0941a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.test.ts @@ -5,7 +5,8 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { buildEventTypeSignal } from './build_event_type_signal'; +import { buildEventTypeSignal, isEventTypeSignal } from './build_event_type_signal'; +import { BaseSignalHit } from './types'; describe('buildEventTypeSignal', () => { beforeEach(() => { @@ -44,4 +45,57 @@ describe('buildEventTypeSignal', () => { }; expect(eventType).toEqual(expected); }); + + test('It validates a sample doc with no signal type as "false"', () => { + const doc = sampleDocNoSortId(); + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates a sample doc with a signal type as "true"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: { + rule: { id: 'id-123' }, + }, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(true); + }); + + test('It validates a numeric signal string as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: 'something', + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates an empty object as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: {}, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); + + test('It validates an empty rule object as "false"', () => { + const doc: BaseSignalHit = ({ + ...sampleDocNoSortId(), + _source: { + ...sampleDocNoSortId()._source, + signal: { + rule: {}, + }, + }, + } as unknown) as BaseSignalHit; + expect(isEventTypeSignal(doc)).toEqual(false); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts index 81c9d1dedcc56..3d78cf5ce5e46 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_event_type_signal.ts @@ -13,3 +13,16 @@ export const buildEventTypeSignal = (doc: BaseSignalHit): object => { return { kind: 'signal' }; } }; + +/** + * Given a document this will return true if that document is a signal + * document. We can't guarantee the code will call this function with a document + * before adding the _source.event.kind = "signal" from "buildEventTypeSignal" + * so we do basic testing to ensure that if the object has the fields of: + * "signal.rule.id" then it will be one of our signals rather than a customer + * overwritten signal. + * @param doc The document which might be a signal or it might be a regular log + */ +export const isEventTypeSignal = (doc: BaseSignalHit): boolean => { + return doc._source.signal?.rule?.id != null && typeof doc._source.signal?.rule?.id === 'string'; +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts index d0c451bbdf2e2..c5e6bc9f157e0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.test.ts @@ -5,8 +5,14 @@ */ import { sampleDocNoSortId } from './__mocks__/es_results'; -import { buildSignal, buildParent, buildAncestors, additionalSignalFields } from './build_signal'; -import { Signal, Ancestor } from './types'; +import { + buildSignal, + buildParent, + buildAncestors, + additionalSignalFields, + removeClashes, +} from './build_signal'; +import { Signal, Ancestor, BaseSignalHit } from './types'; import { getRulesSchemaMock, ANCHOR_DATE, @@ -302,4 +308,64 @@ describe('buildSignal', () => { ]; expect(signal).toEqual(expected); }); + + describe('removeClashes', () => { + test('it will call renameClashes with a regular doc and not mutate it if it does not have a signal clash', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const output = removeClashes(doc); + expect(output).toBe(doc); // reference check + }); + + test('it will call renameClashes with a regular doc and not change anything', () => { + const doc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const output = removeClashes(doc); + expect(output).toEqual(doc); // deep equal check + }); + + test('it will remove a "signal" numeric clash', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: 127, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')); + }); + + test('it will remove a "signal" object clash', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { child_1: { child_2: 'Test nesting' } }, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + expect(output).toEqual(sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')); + }); + + test('it will not remove a "signal" if that is signal is one of our signals', () => { + const sampleDoc = sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'); + const doc = ({ + ...sampleDoc, + _source: { + ...sampleDoc._source, + signal: { rule: { id: '123' } }, + }, + } as unknown) as BaseSignalHit; + const output = removeClashes(doc); + const expected = { + ...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71'), + _source: { + ...sampleDocNoSortId('d5e8eb51-a6a0-456d-8a15-4b79bfec3d71')._source, + signal: { rule: { id: '123' } }, + }, + }; + expect(output).toEqual(expected); + }); + }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts index 947938de6caca..b36a1cbb4a6b3 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/build_signal.ts @@ -5,6 +5,7 @@ */ import { RulesSchema } from '../../../../common/detection_engine/schemas/response/rules_schema'; +import { isEventTypeSignal } from './build_event_type_signal'; import { Signal, Ancestor, BaseSignalHit } from './types'; /** @@ -48,15 +49,37 @@ export const buildAncestors = (doc: BaseSignalHit): Ancestor[] => { } }; +/** + * This removes any signal named clashes such as if a source index has + * "signal" but is not a signal object we put onto the object. If this + * is our "signal object" then we don't want to remove it. + * @param doc The source index doc to a signal. + */ +export const removeClashes = (doc: BaseSignalHit): BaseSignalHit => { + const { signal, ...noSignal } = doc._source; + if (signal == null || isEventTypeSignal(doc)) { + return doc; + } else { + return { + ...doc, + _source: { ...noSignal }, + }; + } +}; + /** * Builds the `signal.*` fields that are common across all signals. * @param docs The parent signals/events of the new signal to be built. * @param rule The rule that is generating the new signal. */ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => { - const parents = docs.map(buildParent); + const removedClashes = docs.map(removeClashes); + const parents = removedClashes.map(buildParent); const depth = parents.reduce((acc, parent) => Math.max(parent.depth, acc), 0) + 1; - const ancestors = docs.reduce((acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), []); + const ancestors = removedClashes.reduce( + (acc: Ancestor[], doc) => acc.concat(buildAncestors(doc)), + [] + ); return { parents, ancestors, @@ -72,9 +95,11 @@ export const buildSignal = (docs: BaseSignalHit[], rule: RulesSchema): Signal => */ export const additionalSignalFields = (doc: BaseSignalHit) => { return { - parent: buildParent(doc), + parent: buildParent(removeClashes(doc)), original_time: doc._source['@timestamp'], original_event: doc._source.event ?? undefined, threshold_count: doc._source.threshold_count ?? undefined, + original_signal: + doc._source.signal != null && !isEventTypeSignal(doc) ? doc._source.signal : undefined, }; }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts index b7cc13fd13a01..eeeda6561892d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts @@ -330,6 +330,18 @@ describe('singleBulkCreate', () => { ]); }); + test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own numeric signal type', () => { + const doc = { ...sampleDocWithAncestors(), _source: { signal: 1234 } }; + const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); + expect(filtered).toEqual([]); + }); + + test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own object signal type', () => { + const doc = { ...sampleDocWithAncestors(), _source: { signal: {} } }; + const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); + expect(filtered).toEqual([]); + }); + test('create successful and returns proper createdItemsCount', async () => { const sampleParams = sampleRuleAlertParams(); mockService.callCluster.mockResolvedValue(sampleBulkCreateDuplicateResult); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts index 759890cc9d074..d8889dcfcf471 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts @@ -14,6 +14,7 @@ import { generateId, makeFloatString, errorAggregator } from './utils'; import { buildBulkBody } from './build_bulk_body'; import { BuildRuleMessage } from './rule_messages'; import { Logger } from '../../../../../../../src/core/server'; +import { isEventTypeSignal } from './build_event_type_signal'; interface SingleBulkCreateParams { filteredEvents: SignalSearchResponse; @@ -50,7 +51,7 @@ export const filterDuplicateRules = ( signalSearchResponse: SignalSearchResponse ) => { return signalSearchResponse.hits.hits.filter((doc) => { - if (doc._source.signal == null) { + if (doc._source.signal == null || !isEventTypeSignal(doc)) { return true; } else { return !( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 9d4e7d8a81051..7128feb80ab3c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -157,6 +157,7 @@ export interface Signal { original_event?: SearchTypes; status: Status; threshold_count?: SearchTypes; + original_signal?: SearchTypes; depth: number; } diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts new file mode 100644 index 0000000000000..0cd1c21447dfe --- /dev/null +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/generating_signals.ts @@ -0,0 +1,508 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import expect from '@kbn/expect'; + +import { CreateRulesSchema } from '../../../../plugins/security_solution/common/detection_engine/schemas/request'; +import { DEFAULT_SIGNALS_INDEX } from '../../../../plugins/security_solution/common/constants'; +import { FtrProviderContext } from '../../common/ftr_provider_context'; +import { + createRule, + createSignalsIndex, + deleteAllAlerts, + deleteSignalsIndex, + getAllSignals, + getSignalsByRuleIds, + getSimpleRule, + waitForSignalsToBePresent, +} from '../../utils'; + +/** + * Specific _id to use for some of the tests. If the archiver changes and you see errors + * here, update this to a new value of a chosen auditbeat record and update the tests values. + */ +export const ID = 'BhbXBmkBR346wHgn4PeZ'; + +// eslint-disable-next-line import/no-default-export +export default ({ getService }: FtrProviderContext) => { + const supertest = getService('supertest'); + const esArchiver = getService('esArchiver'); + const es = getService('es'); + + describe('Generating signals from source indexes', () => { + beforeEach(async () => { + await deleteAllAlerts(es); + await createSignalsIndex(supertest); + }); + + afterEach(async () => { + await deleteSignalsIndex(supertest); + await deleteAllAlerts(es); + }); + + describe('Signals from audit beat are of the expected structure', () => { + beforeEach(async () => { + await esArchiver.load('auditbeat/hosts'); + }); + + afterEach(async () => { + await esArchiver.unload('auditbeat/hosts'); + }); + + it('should have the specific audit record for _id or none of these tests below will pass', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + from: '1900-01-01T00:00:00.000Z', + query: `_id:${ID}`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits.length).greaterThan(0); + }); + + it('should have recorded the rule_id within the signal', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + from: '1900-01-01T00:00:00.000Z', + query: `_id:${ID}`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits[0]._source.signal.rule.rule_id).eql(getSimpleRule().rule_id); + }); + + it('should query and get back expected signal structure using a basic KQL query', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + from: '1900-01-01T00:00:00.000Z', + query: `_id:${ID}`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + expect(signalNoRule).eql({ + parents: [ + { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + ], + ancestors: [ + { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + ], + status: 'open', + depth: 1, + parent: { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + original_time: '2019-02-19T17:40:03.790Z', + original_event: { + action: 'socket_closed', + dataset: 'socket', + kind: 'event', + module: 'system', + }, + }); + }); + + it('should query and get back expected signal structure when it is a signal on a signal', async () => { + // create a 1 signal from 1 auditbeat record + const rule: CreateRulesSchema = { + ...getSimpleRule(), + from: '1900-01-01T00:00:00.000Z', + query: `_id:${ID}`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + + // Run signals on top of that 1 signal which should create a single signal (on top of) a signal + const ruleForSignals: CreateRulesSchema = { + ...getSimpleRule(), + rule_id: 'signal-on-signal', + index: [`${DEFAULT_SIGNALS_INDEX}*`], + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + }; + await createRule(supertest, ruleForSignals); + await waitForSignalsToBePresent(supertest, 2); + + // Get our single signal on top of a signal + const signalsOpen = await getSignalsByRuleIds(supertest, ['signal-on-signal']); + + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + expect(signalNoRule).eql({ + parents: [ + { + rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it + id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + ancestors: [ + { + id: 'BhbXBmkBR346wHgn4PeZ', + type: 'event', + index: 'auditbeat-8.0.0-2019.02.19-000001', + depth: 0, + }, + { + rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it + id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + status: 'open', + depth: 2, + parent: { + rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it + id: 'acf538fc082adf970012be166527c4d9fc120f0015f145e0a466a3ceb32db606', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + original_time: signalNoRule.original_time, // original_time will always be changing sine it's based on a signal created here, so skip testing it + original_event: { + action: 'socket_closed', + dataset: 'socket', + kind: 'signal', + module: 'system', + }, + }); + }); + }); + + /** + * These are a set of tests for whenever someone sets up their source + * index to have a name and mapping clash against "signal" with a numeric value. + * You should see the "signal" name/clash being copied to "original_signal" + * underneath the signal object and no errors when they do have a clash. + */ + describe('Signals generated from name clashes', () => { + beforeEach(async () => { + await esArchiver.load('signals/numeric_name_clash'); + }); + + afterEach(async () => { + await esArchiver.unload('signals/numeric_name_clash'); + }); + + it('should have the specific audit record for _id or none of these tests below will pass', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_name_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits.length).greaterThan(0); + }); + + it('should have recorded the rule_id within the signal', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_name_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits[0]._source.signal.rule.rule_id).eql(getSimpleRule().rule_id); + }); + + it('should query and get back expected signal structure using a basic KQL query', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_name_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + expect(signalNoRule).eql({ + parents: [ + { + id: '1', + type: 'event', + index: 'signal_name_clash', + depth: 0, + }, + ], + ancestors: [ + { + id: '1', + type: 'event', + index: 'signal_name_clash', + depth: 0, + }, + ], + status: 'open', + depth: 1, + parent: { + id: '1', + type: 'event', + index: 'signal_name_clash', + depth: 0, + }, + original_time: '2020-10-28T05:08:53.000Z', + original_signal: 1, + }); + }); + + it('should query and get back expected signal structure when it is a signal on a signal', async () => { + // create a 1 signal from 1 auditbeat record + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_name_clash'], + from: '1900-01-01T00:00:00.000Z', + query: `_id:1`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + + // Run signals on top of that 1 signal which should create a single signal (on top of) a signal + const ruleForSignals: CreateRulesSchema = { + ...getSimpleRule(), + rule_id: 'signal-on-signal', + index: [`${DEFAULT_SIGNALS_INDEX}*`], + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + }; + await createRule(supertest, ruleForSignals); + await waitForSignalsToBePresent(supertest, 2); + + // Get our single signal on top of a signal + const signalsOpen = await getSignalsByRuleIds(supertest, ['signal-on-signal']); + + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + + expect(signalNoRule).eql({ + parents: [ + { + rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it + id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + ancestors: [ + { + id: '1', + type: 'event', + index: 'signal_name_clash', + depth: 0, + }, + { + rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it + id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + status: 'open', + depth: 2, + parent: { + rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it + id: 'b63bcc90b9393f94899991397a3c2df2f3f5c6ebf56440434500f1e1419df7c9', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + original_time: signalNoRule.original_time, // original_time will always be changing sine it's based on a signal created here, so skip testing it + original_event: { + kind: 'signal', + }, + }); + }); + }); + + /** + * These are a set of tests for whenever someone sets up their source + * index to have a name and mapping clash against "signal" with an object value. + * You should see the "signal" object/clash being copied to "original_signal" underneath + * the signal object and no errors when they do have a clash. + */ + describe('Signals generated from name clashes', () => { + beforeEach(async () => { + await esArchiver.load('signals/object_clash'); + }); + + afterEach(async () => { + await esArchiver.unload('signals/object_clash'); + }); + + it('should have the specific audit record for _id or none of these tests below will pass', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_object_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits.length).greaterThan(0); + }); + + it('should have recorded the rule_id within the signal', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_object_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + expect(signalsOpen.hits.hits[0]._source.signal.rule.rule_id).eql(getSimpleRule().rule_id); + }); + + it('should query and get back expected signal structure using a basic KQL query', async () => { + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_object_clash'], + from: '1900-01-01T00:00:00.000Z', + query: '_id:1', + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + const signalsOpen = await getAllSignals(supertest); + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + expect(signalNoRule).eql({ + parents: [ + { + id: '1', + type: 'event', + index: 'signal_object_clash', + depth: 0, + }, + ], + ancestors: [ + { + id: '1', + type: 'event', + index: 'signal_object_clash', + depth: 0, + }, + ], + status: 'open', + depth: 1, + parent: { + id: '1', + type: 'event', + index: 'signal_object_clash', + depth: 0, + }, + original_time: '2020-10-28T05:08:53.000Z', + original_signal: { + child_1: { + child_2: { + value: 'some_value', + }, + }, + }, + }); + }); + + it('should query and get back expected signal structure when it is a signal on a signal', async () => { + // create a 1 signal from 1 auditbeat record + const rule: CreateRulesSchema = { + ...getSimpleRule(), + index: ['signal_object_clash'], + from: '1900-01-01T00:00:00.000Z', + query: `_id:1`, + }; + await createRule(supertest, rule); + await waitForSignalsToBePresent(supertest, 1); + + // Run signals on top of that 1 signal which should create a single signal (on top of) a signal + const ruleForSignals: CreateRulesSchema = { + ...getSimpleRule(), + rule_id: 'signal-on-signal', + index: [`${DEFAULT_SIGNALS_INDEX}*`], + from: '1900-01-01T00:00:00.000Z', + query: '*:*', + }; + await createRule(supertest, ruleForSignals); + await waitForSignalsToBePresent(supertest, 2); + + // Get our single signal on top of a signal + const signalsOpen = await getSignalsByRuleIds(supertest, ['signal-on-signal']); + + // remove rule to cut down on touch points for test changes when the rule format changes + const { rule: removedRule, ...signalNoRule } = signalsOpen.hits.hits[0]._source.signal; + + expect(signalNoRule).eql({ + parents: [ + { + rule: signalNoRule.parents[0].rule, // rule id is always changing so skip testing it + id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + ancestors: [ + { + id: '1', + type: 'event', + index: 'signal_object_clash', + depth: 0, + }, + { + rule: signalNoRule.ancestors[1].rule, // rule id is always changing so skip testing it + id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + ], + status: 'open', + depth: 2, + parent: { + rule: signalNoRule.parent?.rule, // parent.rule is always changing so skip testing it + id: 'd2114ed6553816f87d6707b5bc50b88751db73b0f4930433d0890474804aa179', + type: 'signal', + index: '.siem-signals-default-000001', + depth: 1, + }, + original_time: signalNoRule.original_time, // original_time will always be changing sine it's based on a signal created here, so skip testing it + original_event: { + kind: 'signal', + }, + }); + }); + }); + }); +}; diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts index 24b76853164f2..962ae53b1241f 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/tests/index.ts @@ -22,6 +22,7 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./export_rules')); loadTestFile(require.resolve('./find_rules')); loadTestFile(require.resolve('./find_statuses')); + loadTestFile(require.resolve('./generating_signals')); loadTestFile(require.resolve('./get_prepackaged_rules_status')); loadTestFile(require.resolve('./import_rules')); loadTestFile(require.resolve('./read_rules')); diff --git a/x-pack/test/detection_engine_api_integration/utils.ts b/x-pack/test/detection_engine_api_integration/utils.ts index 05a0f73dd0dc4..c5e417c710283 100644 --- a/x-pack/test/detection_engine_api_integration/utils.ts +++ b/x-pack/test/detection_engine_api_integration/utils.ts @@ -143,6 +143,19 @@ export const getQuerySignalIds = (signalIds: SignalIds) => ({ }, }); +/** + * Given an array of ruleIds for a test this will get the signals + * created from that rule_id. + * @param ruleIds The rule_id to search for signals + */ +export const getQuerySignalsRuleId = (ruleIds: string[]) => ({ + query: { + terms: { + 'signal.rule.rule_id': ruleIds, + }, + }, +}); + export const setSignalStatus = ({ signalIds, status, @@ -834,6 +847,22 @@ export const getAllSignals = async ( return signalsOpen; }; +export const getSignalsByRuleIds = async ( + supertest: SuperTest, + ruleIds: string[] +): Promise< + SearchResponse<{ + signal: Signal; + }> +> => { + const { body: signalsOpen }: { body: SearchResponse<{ signal: Signal }> } = await supertest + .post(DETECTION_ENGINE_QUERY_SIGNALS_URL) + .set('kbn-xsrf', 'true') + .send(getQuerySignalsRuleId(ruleIds)) + .expect(200); + return signalsOpen; +}; + export const installPrePackagedRules = async ( supertest: SuperTest ): Promise => { diff --git a/x-pack/test/functional/es_archives/signals/numeric_name_clash/data.json b/x-pack/test/functional/es_archives/signals/numeric_name_clash/data.json new file mode 100644 index 0000000000000..ca2cf0de2d845 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/numeric_name_clash/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "signal_name_clash", + "source": { + "@timestamp": "2020-10-28T05:08:53.000Z", + "signal": 1 + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/numeric_name_clash/mappings.json b/x-pack/test/functional/es_archives/signals/numeric_name_clash/mappings.json new file mode 100644 index 0000000000000..98823cb4b4250 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/numeric_name_clash/mappings.json @@ -0,0 +1,20 @@ +{ + "type": "index", + "value": { + "index": "signal_name_clash", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { "type": "keyword" } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +} diff --git a/x-pack/test/functional/es_archives/signals/object_clash/data.json b/x-pack/test/functional/es_archives/signals/object_clash/data.json new file mode 100644 index 0000000000000..d2f844312e8fb --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/object_clash/data.json @@ -0,0 +1,12 @@ +{ + "type": "doc", + "value": { + "id": "1", + "index": "signal_object_clash", + "source": { + "@timestamp": "2020-10-28T05:08:53.000Z", + "signal": { "child_1": { "child_2": { "value": "some_value" } } } + }, + "type": "_doc" + } +} diff --git a/x-pack/test/functional/es_archives/signals/object_clash/mappings.json b/x-pack/test/functional/es_archives/signals/object_clash/mappings.json new file mode 100644 index 0000000000000..9297ff3e867c9 --- /dev/null +++ b/x-pack/test/functional/es_archives/signals/object_clash/mappings.json @@ -0,0 +1,20 @@ +{ + "type": "index", + "value": { + "index": "signal_object_clash", + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "signal": { "type": "object" } + } + }, + "settings": { + "index": { + "number_of_replicas": "1", + "number_of_shards": "1" + } + } + } +}