From 4d921ffb7e8cfd5de5be6ed91daf336f45194dd3 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 15 Jun 2021 16:59:50 -0500 Subject: [PATCH] [Security Solution][CTI] Event enrichment search strategy (#101553) * Adding boilerplate for new CTI search strategy type This is going to be a subtype of the general SecSol search strategy; the main functionality is going to be: * transformation of the incoming parameters into named equivalents * transformation of responses to include enrichment context fields (matched.*) * More boilerplate, including tests A few type errors because our functions don't actually do anything yet, nor are our request/response types fleshed out. * Starting to flesh out the request parsing * Defines a basic request, along with a mock * Defines helper function to generate should clauses from field values * Adds placeholder tests throughout * Fleshing out unit tests around our enrichment query * Fleshing out response parsing of eventEnrichment strategy * Fix types from elasticsearch Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../security_solution/common/cti/constants.ts | 13 ++ .../common/ecs/threat/index.ts | 2 + .../security_solution/cti/index.mock.ts | 110 +++++++++++ .../security_solution/cti/index.ts | 26 +++ .../security_solution/index.ts | 10 + .../factory/cti/event_enrichment/factory.ts | 16 ++ .../cti/event_enrichment/helpers.test.ts | 172 ++++++++++++++++++ .../factory/cti/event_enrichment/helpers.ts | 83 +++++++++ .../factory/cti/event_enrichment/index.ts | 8 + .../cti/event_enrichment/query.test.ts | 90 +++++++++ .../factory/cti/event_enrichment/query.ts | 52 ++++++ .../cti/event_enrichment/response.test.ts | 89 +++++++++ .../factory/cti/event_enrichment/response.ts | 31 ++++ .../security_solution/factory/cti/index.ts | 15 ++ .../security_solution/factory/index.ts | 6 +- 15 files changed, 721 insertions(+), 2 deletions(-) create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts create mode 100644 x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts create mode 100644 x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/index.ts diff --git a/x-pack/plugins/security_solution/common/cti/constants.ts b/x-pack/plugins/security_solution/common/cti/constants.ts index 3423f17e3f6833..10452996eae6f8 100644 --- a/x-pack/plugins/security_solution/common/cti/constants.ts +++ b/x-pack/plugins/security_solution/common/cti/constants.ts @@ -44,3 +44,16 @@ export const SORTED_THREAT_SUMMARY_FIELDS = [ INDICATOR_FIRSTSEEN, INDICATOR_LASTSEEN, ]; + +export const EVENT_ENRICHMENT_INDICATOR_FIELD_MAP = { + 'file.hash.md5': 'threatintel.indicator.file.hash.md5', + 'file.hash.sha1': 'threatintel.indicator.file.hash.sha1', + 'file.hash.sha256': 'threatintel.indicator.file.hash.sha256', + 'file.pe.imphash': 'threatintel.indicator.file.pe.imphash', + 'file.elf.telfhash': 'threatintel.indicator.file.elf.telfhash', + 'file.hash.ssdeep': 'threatintel.indicator.file.hash.ssdeep', + 'source.ip': 'threatintel.indicator.ip', + 'destination.ip': 'threatintel.indicator.ip', + 'url.full': 'threatintel.indicator.url.full', + 'registry.path': 'threatintel.indicator.registry.path', +}; diff --git a/x-pack/plugins/security_solution/common/ecs/threat/index.ts b/x-pack/plugins/security_solution/common/ecs/threat/index.ts index 19923a82dc846f..e5e7964c5d09d6 100644 --- a/x-pack/plugins/security_solution/common/ecs/threat/index.ts +++ b/x-pack/plugins/security_solution/common/ecs/threat/index.ts @@ -10,6 +10,8 @@ import { EventEcs } from '../event'; interface ThreatMatchEcs { atomic?: string[]; field?: string[]; + id?: string[]; + index?: string[]; type?: string[]; } diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts new file mode 100644 index 00000000000000..f3dee5a21e4c98 --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.mock.ts @@ -0,0 +1,110 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEsSearchResponse } from 'src/plugins/data/public'; + +import { + CtiEventEnrichmentRequestOptions, + CtiEventEnrichmentStrategyResponse, + CtiQueries, +} from '.'; + +export const buildEventEnrichmentRequestOptionsMock = ( + overrides: Partial = {} +): CtiEventEnrichmentRequestOptions => ({ + defaultIndex: ['filebeat-*'], + eventFields: { + 'file.hash.md5': '1eee2bf3f56d8abed72da2bc523e7431', + 'source.ip': '127.0.0.1', + 'url.full': 'elastic.co', + }, + factoryQueryType: CtiQueries.eventEnrichment, + filterQuery: '{"bool":{"must":[],"filter":[{"match_all":{}}],"should":[],"must_not":[]}}', + timerange: { interval: '', from: '2020-09-13T09:00:43.249Z', to: '2020-09-14T09:00:43.249Z' }, + ...overrides, +}); + +export const buildEventEnrichmentRawResponseMock = (): IEsSearchResponse => ({ + rawResponse: { + took: 17, + timed_out: false, + _shards: { + total: 1, + successful: 1, + skipped: 0, + failed: 0, + }, + hits: { + total: { + value: 1, + relation: 'eq', + }, + max_score: 6.0637846, + hits: [ + { + _index: 'filebeat-8.0.0-2021.05.28-000001', + _id: '31408415b6d5601a92d29b86c2519658f210c194057588ae396d55cc20b3f03d', + _score: 6.0637846, + fields: { + 'event.category': ['threat'], + 'threatintel.indicator.file.type': ['html'], + 'related.hash': [ + '5529de7b60601aeb36f57824ed0e1ae8', + '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e', + '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p', + ], + 'threatintel.indicator.first_seen': ['2021-05-28T18:33:29.000Z'], + 'threatintel.indicator.file.hash.tlsh': [ + 'FFB20B82F6617061C32784E2712F7A46B179B04FD1EA54A0F28CD8E9CFE4CAA1617F1C', + ], + 'service.type': ['threatintel'], + 'threatintel.indicator.file.hash.ssdeep': [ + '768:NXSFGJ/ooP6FawrB7Bo1MWnF/jRmhJImp:1SFXIqBo1Mwj2p', + ], + 'agent.type': ['filebeat'], + 'event.module': ['threatintel'], + 'threatintel.indicator.type': ['file'], + 'agent.name': ['rylastic.local'], + 'threatintel.indicator.file.hash.sha256': [ + '15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e', + ], + 'event.kind': ['enrichment'], + 'threatintel.indicator.file.hash.md5': ['5529de7b60601aeb36f57824ed0e1ae8'], + 'fileset.name': ['abusemalware'], + 'input.type': ['httpjson'], + 'agent.hostname': ['rylastic.local'], + tags: ['threatintel-abusemalware', 'forwarded'], + 'event.ingested': ['2021-05-28T18:33:55.086Z'], + '@timestamp': ['2021-05-28T18:33:52.993Z'], + 'agent.id': ['ff93aee5-86a1-4a61-b0e6-0cdc313d01b5'], + 'ecs.version': ['1.6.0'], + 'event.reference': [ + 'https://urlhaus-api.abuse.ch/v1/download/15b012e6f626d0f88c2926d2bf4ca394d7b8ee07cc06d2ec05ea76bed3e8a05e/', + ], + 'event.type': ['indicator'], + 'event.created': ['2021-05-28T18:33:52.993Z'], + 'agent.ephemeral_id': ['d6b14f65-5bf3-430d-8315-7b5613685979'], + 'threatintel.indicator.file.size': [24738], + 'agent.version': ['8.0.0'], + 'event.dataset': ['threatintel.abusemalware'], + }, + matched_queries: ['file.hash.md5'], + }, + ], + }, + }, +}); + +export const buildEventEnrichmentResponseMock = ( + overrides: Partial = {} +): CtiEventEnrichmentStrategyResponse => ({ + ...buildEventEnrichmentRawResponseMock(), + enrichments: [], + inspect: { dsl: ['{"mocked": "json"}'] }, + totalCount: 0, + ...overrides, +}); diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts new file mode 100644 index 00000000000000..788a44bc5b9f7f --- /dev/null +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/cti/index.ts @@ -0,0 +1,26 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { IEsSearchResponse } from 'src/plugins/data/public'; +import { Inspect } from '../../common'; +import { RequestBasicOptions } from '..'; + +export enum CtiQueries { + eventEnrichment = 'eventEnrichment', +} + +export interface CtiEventEnrichmentRequestOptions extends RequestBasicOptions { + eventFields: Record; +} + +export type CtiEnrichment = Record; + +export interface CtiEventEnrichmentStrategyResponse extends IEsSearchResponse { + enrichments: CtiEnrichment[]; + inspect?: Inspect; + totalCount: number; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts index 956b785079d8df..06d4a16699b8f4 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/index.ts @@ -66,6 +66,11 @@ import { MatrixHistogramStrategyResponse, } from './matrix_histogram'; import { TimerangeInput, SortField, PaginationInput, PaginationInputPaginated } from '../common'; +import { + CtiEventEnrichmentRequestOptions, + CtiEventEnrichmentStrategyResponse, + CtiQueries, +} from './cti'; export * from './hosts'; export * from './matrix_histogram'; @@ -76,6 +81,7 @@ export type FactoryQueryTypes = | HostsKpiQueries | NetworkQueries | NetworkKpiQueries + | CtiQueries | typeof MatrixHistogramQuery | typeof MatrixHistogramQueryEntities; @@ -145,6 +151,8 @@ export type StrategyResponseType = T extends HostsQ ? NetworkKpiUniquePrivateIpsStrategyResponse : T extends typeof MatrixHistogramQuery ? MatrixHistogramStrategyResponse + : T extends CtiQueries.eventEnrichment + ? CtiEventEnrichmentStrategyResponse : never; export type StrategyRequestType = T extends HostsQueries.hosts @@ -193,6 +201,8 @@ export type StrategyRequestType = T extends HostsQu ? NetworkKpiUniquePrivateIpsRequestOptions : T extends typeof MatrixHistogramQuery ? MatrixHistogramRequestOptions + : T extends CtiQueries.eventEnrichment + ? CtiEventEnrichmentRequestOptions : never; export interface DocValueFieldsInput { diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts new file mode 100644 index 00000000000000..e5e1a14df3c1c8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/factory.ts @@ -0,0 +1,16 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; +import { SecuritySolutionFactory } from '../../types'; +import { buildEventEnrichmentQuery } from './query'; +import { parseEventEnrichmentResponse } from './response'; + +export const eventEnrichment: SecuritySolutionFactory = { + buildDsl: buildEventEnrichmentQuery, + parse: parseEventEnrichmentResponse, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts new file mode 100644 index 00000000000000..a246b66d462cef --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.test.ts @@ -0,0 +1,172 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildIndicatorEnrichments, buildIndicatorShouldClauses, getTotalCount } from './helpers'; + +describe('buildIndicatorShouldClauses', () => { + it('returns an empty array given an empty fieldset', () => { + expect(buildIndicatorShouldClauses({})).toEqual([]); + }); + + it('returns an empty array given no relevant values', () => { + const eventFields = { 'url.domain': 'elastic.co' }; + expect(buildIndicatorShouldClauses(eventFields)).toEqual([]); + }); + + it('returns a clause for each relevant value', () => { + const eventFields = { 'source.ip': '127.0.0.1', 'url.full': 'elastic.co' }; + expect(buildIndicatorShouldClauses(eventFields)).toHaveLength(2); + }); + + it('excludes non-CTI fields', () => { + const eventFields = { 'source.ip': '127.0.0.1', 'url.domain': 'elastic.co' }; + expect(buildIndicatorShouldClauses(eventFields)).toHaveLength(1); + }); + + it('defines a named query where the name is the event field and the value is the event field value', () => { + const eventFields = { 'file.hash.md5': '1eee2bf3f56d8abed72da2bc523e7431' }; + + expect(buildIndicatorShouldClauses(eventFields)).toContainEqual({ + match: { + 'threatintel.indicator.file.hash.md5': { + _name: 'file.hash.md5', + query: '1eee2bf3f56d8abed72da2bc523e7431', + }, + }, + }); + }); + + it('returns valid queries for multiple valid fields', () => { + const eventFields = { 'source.ip': '127.0.0.1', 'url.full': 'elastic.co' }; + expect(buildIndicatorShouldClauses(eventFields)).toEqual( + expect.arrayContaining([ + { match: { 'threatintel.indicator.ip': { _name: 'source.ip', query: '127.0.0.1' } } }, + { match: { 'threatintel.indicator.url.full': { _name: 'url.full', query: 'elastic.co' } } }, + ]) + ); + }); +}); + +describe('getTotalCount', () => { + it('returns 0 when total is null (not tracking)', () => { + expect(getTotalCount(null)).toEqual(0); + }); + + it('returns total when total is a number', () => { + expect(getTotalCount(5)).toEqual(5); + }); + + it('returns total.value when total is an object', () => { + expect(getTotalCount({ value: 20, relation: 'eq' })).toEqual(20); + }); +}); + +describe('buildIndicatorEnrichments', () => { + it('returns nothing if hits have no matched queries', () => { + const hits = [{ _id: '_id', _index: '_index', matched_queries: [] }]; + expect(buildIndicatorEnrichments(hits)).toEqual([]); + }); + + it("returns nothing if hits' matched queries are not valid", () => { + const hits = [{ _id: '_id', _index: '_index', matched_queries: ['invalid.field'] }]; + expect(buildIndicatorEnrichments(hits)).toEqual([]); + }); + + it('builds a single enrichment if the hit has a matched query', () => { + const hits = [ + { + _id: '_id', + _index: '_index', + matched_queries: ['file.hash.md5'], + fields: { + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + }, + }, + ]; + + expect(buildIndicatorEnrichments(hits)).toEqual([ + expect.objectContaining({ + 'matched.atomic': ['indicator_value'], + 'matched.field': ['file.hash.md5'], + 'matched.id': ['_id'], + 'matched.index': ['_index'], + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + }), + ]); + }); + + it('builds multiple enrichments if the hit has matched queries', () => { + const hits = [ + { + _id: '_id', + _index: '_index', + matched_queries: ['file.hash.md5', 'source.ip'], + fields: { + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + 'threatintel.indicator.ip': ['127.0.0.1'], + }, + }, + ]; + + expect(buildIndicatorEnrichments(hits)).toEqual([ + expect.objectContaining({ + 'matched.atomic': ['indicator_value'], + 'matched.field': ['file.hash.md5'], + 'matched.id': ['_id'], + 'matched.index': ['_index'], + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + 'threatintel.indicator.ip': ['127.0.0.1'], + }), + expect.objectContaining({ + 'matched.atomic': ['127.0.0.1'], + 'matched.field': ['source.ip'], + 'matched.id': ['_id'], + 'matched.index': ['_index'], + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + 'threatintel.indicator.ip': ['127.0.0.1'], + }), + ]); + }); + + it('builds an enrichment for each hit', () => { + const hits = [ + { + _id: '_id', + _index: '_index', + matched_queries: ['file.hash.md5'], + fields: { + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + }, + }, + { + _id: '_id2', + _index: '_index2', + matched_queries: ['source.ip'], + fields: { + 'threatintel.indicator.ip': ['127.0.0.1'], + }, + }, + ]; + + expect(buildIndicatorEnrichments(hits)).toEqual([ + expect.objectContaining({ + 'matched.atomic': ['indicator_value'], + 'matched.field': ['file.hash.md5'], + 'matched.id': ['_id'], + 'matched.index': ['_index'], + 'threatintel.indicator.file.hash.md5': ['indicator_value'], + }), + expect.objectContaining({ + 'matched.atomic': ['127.0.0.1'], + 'matched.field': ['source.ip'], + 'matched.id': ['_id2'], + 'matched.index': ['_index2'], + 'threatintel.indicator.ip': ['127.0.0.1'], + }), + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts new file mode 100644 index 00000000000000..e4ed05baeed778 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/helpers.ts @@ -0,0 +1,83 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { get, isEmpty } from 'lodash'; +import { estypes } from '@elastic/elasticsearch'; + +import { EVENT_ENRICHMENT_INDICATOR_FIELD_MAP } from '../../../../../../common/cti/constants'; +import { CtiEnrichment } from '../../../../../../common/search_strategy/security_solution/cti'; + +type EventField = keyof typeof EVENT_ENRICHMENT_INDICATOR_FIELD_MAP; +const validEventFields = Object.keys(EVENT_ENRICHMENT_INDICATOR_FIELD_MAP) as EventField[]; + +const isValidEventField = (field: string): field is EventField => + validEventFields.includes(field as EventField); + +export const buildIndicatorShouldClauses = ( + eventFields: Record +): estypes.QueryDslQueryContainer[] => { + return validEventFields.reduce((shoulds, eventField) => { + const eventFieldValue = eventFields[eventField]; + + if (!isEmpty(eventFieldValue)) { + shoulds.push({ + match: { + [EVENT_ENRICHMENT_INDICATOR_FIELD_MAP[eventField]]: { + query: eventFieldValue, + _name: eventField, + }, + }, + }); + } + + return shoulds; + }, []); +}; + +export const buildIndicatorEnrichments = (hits: estypes.SearchHit[]): CtiEnrichment[] => { + return hits.flatMap(({ matched_queries: matchedQueries, ...hit }) => { + return ( + matchedQueries?.reduce((enrichments, matchedQuery) => { + if (isValidEventField(matchedQuery)) { + enrichments.push({ + ...hit.fields, + ...buildIndicatorMatchedFields(hit, matchedQuery), + }); + } + + return enrichments; + }, []) ?? [] + ); + }); +}; + +const buildIndicatorMatchedFields = ( + hit: estypes.SearchHit, + eventField: EventField +): Record => { + const indicatorField = EVENT_ENRICHMENT_INDICATOR_FIELD_MAP[eventField]; + const atomic = get(hit.fields, indicatorField) as string[]; + + return { + 'matched.atomic': atomic, + 'matched.field': [eventField], + 'matched.id': [hit._id], + 'matched.index': [hit._index], + }; +}; + +export const getTotalCount = (total: number | estypes.SearchTotalHits | null): number => { + if (total == null) { + return 0; + } + + if (typeof total === 'number') { + return total; + } + + return total.value; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts new file mode 100644 index 00000000000000..6884b7b6320cf5 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/index.ts @@ -0,0 +1,8 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +export { eventEnrichment } from './factory'; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts new file mode 100644 index 00000000000000..bc96a387105c68 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.test.ts @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildEventEnrichmentRequestOptionsMock } from '../../../../../../common/search_strategy/security_solution/cti/index.mock'; +import { buildEventEnrichmentQuery } from './query'; + +describe('buildEventEnrichmentQuery', () => { + it('converts each event field/value into a named filter', () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.query?.bool?.should).toEqual( + expect.arrayContaining([ + { + match: { + 'threatintel.indicator.file.hash.md5': { + _name: 'file.hash.md5', + query: '1eee2bf3f56d8abed72da2bc523e7431', + }, + }, + }, + { match: { 'threatintel.indicator.ip': { _name: 'source.ip', query: '127.0.0.1' } } }, + { match: { 'threatintel.indicator.url.full': { _name: 'url.full', query: 'elastic.co' } } }, + ]) + ); + }); + + it('filters on indicator events', () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.query?.bool?.filter).toEqual( + expect.arrayContaining([{ term: { 'event.type': 'indicator' } }]) + ); + }); + + it('includes the specified timerange', () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.query?.bool?.filter).toEqual( + expect.arrayContaining([ + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2020-09-13T09:00:43.249Z', + lte: '2020-09-14T09:00:43.249Z', + }, + }, + }, + ]) + ); + }); + + it('includes specified docvalue_fields', () => { + const docValueFields = [ + { field: '@timestamp', format: 'date_time' }, + { field: 'event.created', format: 'date_time' }, + { field: 'event.end', format: 'date_time' }, + ]; + const options = buildEventEnrichmentRequestOptionsMock({ docValueFields }); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.docvalue_fields).toEqual(expect.arrayContaining(docValueFields)); + }); + + it('requests all fields', () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.fields).toEqual(['*']); + }); + + it('excludes _source', () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const query = buildEventEnrichmentQuery(options); + expect(query.body?._source).toEqual(false); + }); + + it('includes specified filters', () => { + const filterQuery = { + query: 'query_field: query_value', + language: 'kuery', + }; + + const options = buildEventEnrichmentRequestOptionsMock({ filterQuery }); + const query = buildEventEnrichmentQuery(options); + expect(query.body?.query?.bool?.filter).toEqual(expect.arrayContaining([filterQuery])); + }); +}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts new file mode 100644 index 00000000000000..4760e6a227cd3d --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/query.ts @@ -0,0 +1,52 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isEmpty } from 'lodash'; +import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; +import { createQueryFilterClauses } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; +import { buildIndicatorShouldClauses } from './helpers'; + +export const buildEventEnrichmentQuery: SecuritySolutionFactory['buildDsl'] = ({ + defaultIndex, + docValueFields, + eventFields, + filterQuery, + timerange: { from, to }, +}) => { + const filter = [ + ...createQueryFilterClauses(filterQuery), + { term: { 'event.type': 'indicator' } }, + { + range: { + '@timestamp': { + gte: from, + lte: to, + format: 'strict_date_optional_time', + }, + }, + }, + ]; + + return { + allowNoIndices: true, + ignoreUnavailable: true, + index: defaultIndex, + body: { + _source: false, + ...(!isEmpty(docValueFields) && { docvalue_fields: docValueFields }), + fields: ['*'], + query: { + bool: { + should: buildIndicatorShouldClauses(eventFields), + filter, + minimum_should_match: 1, + }, + }, + }, + }; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts new file mode 100644 index 00000000000000..7ced866e0bb5b6 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.test.ts @@ -0,0 +1,89 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + buildEventEnrichmentRequestOptionsMock, + buildEventEnrichmentRawResponseMock, +} from '../../../../../../common/search_strategy/security_solution/cti/index.mock'; +import { parseEventEnrichmentResponse } from './response'; + +describe('parseEventEnrichmentResponse', () => { + it('includes an accurate inspect response', async () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const response = buildEventEnrichmentRawResponseMock(); + const parsedResponse = await parseEventEnrichmentResponse(options, response); + + const expectedInspect = expect.objectContaining({ + allowNoIndices: true, + body: { + _source: false, + fields: ['*'], + query: { + bool: { + filter: [ + { bool: { filter: [{ match_all: {} }], must: [], must_not: [], should: [] } }, + { term: { 'event.type': 'indicator' } }, + { + range: { + '@timestamp': { + format: 'strict_date_optional_time', + gte: '2020-09-13T09:00:43.249Z', + lte: '2020-09-14T09:00:43.249Z', + }, + }, + }, + ], + minimum_should_match: 1, + should: [ + { + match: { + 'threatintel.indicator.file.hash.md5': { + _name: 'file.hash.md5', + query: '1eee2bf3f56d8abed72da2bc523e7431', + }, + }, + }, + { match: { 'threatintel.indicator.ip': { _name: 'source.ip', query: '127.0.0.1' } } }, + { + match: { + 'threatintel.indicator.url.full': { _name: 'url.full', query: 'elastic.co' }, + }, + }, + ], + }, + }, + }, + ignoreUnavailable: true, + index: ['filebeat-*'], + }); + const parsedInspect = JSON.parse(parsedResponse.inspect!.dsl[0]); + expect(parsedInspect).toEqual(expectedInspect); + }); + + it('includes an accurate total count', async () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const response = buildEventEnrichmentRawResponseMock(); + const parsedResponse = await parseEventEnrichmentResponse(options, response); + + expect(parsedResponse.totalCount).toEqual(1); + }); + + it('adds matched.* enrichment fields based on the named query', async () => { + const options = buildEventEnrichmentRequestOptionsMock(); + const response = buildEventEnrichmentRawResponseMock(); + const parsedResponse = await parseEventEnrichmentResponse(options, response); + + expect(parsedResponse.enrichments).toEqual([ + expect.objectContaining({ + 'matched.atomic': ['5529de7b60601aeb36f57824ed0e1ae8'], + 'matched.field': ['file.hash.md5'], + 'matched.id': ['31408415b6d5601a92d29b86c2519658f210c194057588ae396d55cc20b3f03d'], + 'matched.index': ['filebeat-8.0.0-2021.05.28-000001'], + }), + ]); + }); +}); diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts new file mode 100644 index 00000000000000..29a842d84558c7 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/event_enrichment/response.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { CtiQueries } from '../../../../../../common/search_strategy/security_solution/cti'; +import { inspectStringifyObject } from '../../../../../utils/build_query'; +import { SecuritySolutionFactory } from '../../types'; +import { buildIndicatorEnrichments, getTotalCount } from './helpers'; +import { buildEventEnrichmentQuery } from './query'; + +export const parseEventEnrichmentResponse: SecuritySolutionFactory['parse'] = async ( + options, + response, + deps +) => { + const inspect = { + dsl: [inspectStringifyObject(buildEventEnrichmentQuery(options))], + }; + const totalCount = getTotalCount(response.rawResponse.hits.total); + const enrichments = buildIndicatorEnrichments(response.rawResponse.hits.hits); + + return { + ...response, + enrichments, + inspect, + totalCount, + }; +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/index.ts new file mode 100644 index 00000000000000..5857a0417239c8 --- /dev/null +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/cti/index.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import type { FactoryQueryTypes } from '../../../../../common/search_strategy/security_solution'; +import { CtiQueries } from '../../../../../common/search_strategy/security_solution/cti'; +import type { SecuritySolutionFactory } from '../types'; +import { eventEnrichment } from './event_enrichment'; + +export const ctiFactoryTypes: Record> = { + [CtiQueries.eventEnrichment]: eventEnrichment, +}; diff --git a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts index 346dd20c89441f..5b54c63408d100 100644 --- a/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts +++ b/x-pack/plugins/security_solution/server/search_strategy/security_solution/factory/index.ts @@ -5,12 +5,13 @@ * 2.0. */ -import { FactoryQueryTypes } from '../../../../common/search_strategy/security_solution'; +import type { FactoryQueryTypes } from '../../../../common/search_strategy/security_solution'; +import type { SecuritySolutionFactory } from './types'; import { hostsFactory } from './hosts'; import { matrixHistogramFactory } from './matrix_histogram'; import { networkFactory } from './network'; -import { SecuritySolutionFactory } from './types'; +import { ctiFactoryTypes } from './cti'; export const securitySolutionFactory: Record< FactoryQueryTypes, @@ -19,4 +20,5 @@ export const securitySolutionFactory: Record< ...hostsFactory, ...matrixHistogramFactory, ...networkFactory, + ...ctiFactoryTypes, };