diff --git a/docs/developer/plugin-list.asciidoc b/docs/developer/plugin-list.asciidoc index ffc918af92514..eee92ba433721 100644 --- a/docs/developer/plugin-list.asciidoc +++ b/docs/developer/plugin-list.asciidoc @@ -485,7 +485,7 @@ using the CURL scripts in the scripts folder. |{kib-repo}blob/{branch}/x-pack/plugins/metrics_entities/README.md[metricsEntities] |This is the metrics and entities plugin where you add can add transforms for your project and group those transforms into modules. You can also re-use existing transforms in your -modules as well. +newly created modules as well. |{kib-repo}blob/{branch}/x-pack/plugins/ml/readme.md[ml] diff --git a/x-pack/plugins/metrics_entities/README.md b/x-pack/plugins/metrics_entities/README.md index 6c711ce4fed82..71ac2730f3831 100755 --- a/x-pack/plugins/metrics_entities/README.md +++ b/x-pack/plugins/metrics_entities/README.md @@ -2,7 +2,7 @@ This is the metrics and entities plugin where you add can add transforms for your project and group those transforms into modules. You can also re-use existing transforms in your -modules as well. +newly created modules as well. ## Turn on experimental flags During at least phase 1 of this development, please add these to your `kibana.dev.yml` file to turn on the feature: @@ -309,16 +309,14 @@ are notes during the phased approach. As we approach production and the feature left over TODO's in the code base. - Add these properties to the route which are: - - disable_transforms/exclude flag to exclude 1 or more transforms within a module, - - pipeline flag, - - Change the REST routes on post to change the indexes for whichever indexes you want - - Unit tests to ensure the data of the mapping.json includes the correct fields such as - _meta, at least one alias, a mapping section, etc... - - Add text/keyword and other things to the mappings (not just keyword maybe?) ... At least review the mappings one more time - - Add a sort of @timestamp to the output destination indexes? - - Add the REST Kibana security based tags if needed and push those to any plugins using this plugin. Something like: tags: ['access:metricsEntities-read'] and ['access:metricsEntities-all'], - - Add schema validation choosing some schema library (io-ts or Kibana Schema or ... ) - - Add unit tests - - Add e2e tests - - Move ui code into this plugin from security_solutions? (maybe?) - - UI code could be within `kibana/packages` instead of in here directly and I think we will be better off. + - disable_transforms/exclude flag to exclude 1 or more transforms within a module + - pipeline flag + - Change the REST routes on post to change the indexes for whichever indexes you want +- Unit tests to ensure the data of the mapping.json includes the correct fields such as _meta, at least one alias, a mapping section, etc... +- Add text/keyword and other things to the mappings (not just keyword maybe?) ... At least review the mappings one more time +- Add a sort of @timestamp to the output destination indexes? +- Add the REST Kibana security based tags if needed and push those to any plugins using this plugin. Something like: tags: ['access:metricsEntities-read'] and ['access:metricsEntities-all'], +- Add schema validation choosing some schema library (io-ts or Kibana Schema or ... ) +- Add unit tests +- Add e2e tests +- Any UI code should not end up here. There is none right now, but all UI code should be within a kbn package or security_solutions diff --git a/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.test.ts new file mode 100644 index 0000000000000..8dc4210494ea3 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.test.ts @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { adjustTimeRange } from './adjust_timerange'; +import moment from 'moment'; + +/** Get the return type of adjustTimeRange for TypeScript checks against expected */ +type ReturnTypeAdjustTimeRange = ReturnType; + +describe('adjust_timerange', () => { + beforeEach(() => { + // Adds extra switch to suppress deprecation warnings that moment does not expose in TypeScript + (moment as typeof moment & { + suppressDeprecationWarnings: boolean; + }).suppressDeprecationWarnings = true; + }); + + afterEach(() => { + // Adds extra switch to suppress deprecation warnings that moment does not expose in TypeScript + (moment as typeof moment & { + suppressDeprecationWarnings: boolean; + }).suppressDeprecationWarnings = false; + }); + + test('it will adjust the time range from by rounding down by an hour within "from"', () => { + expect( + adjustTimeRange({ + interval: '5m', + to: '2021-07-06T22:07:56.972Z', + from: '2021-07-06T22:07:56.972Z', + }) + ).toMatchObject>({ + timeRangeAdjusted: { + interval: '5m', + to: '2021-07-06T22:07:56.972Z', + from: '2021-07-06T22:00:00.000Z', // <-- Rounded down by an hour + }, + }); + }); + + test('it will compute the duration between to and and from', () => { + expect( + adjustTimeRange({ + interval: '5m', + to: '2021-07-06T22:08:56.972Z', + from: '2021-07-06T22:07:56.972Z', + }).duration?.asMinutes() + ).toEqual(1); + }); + + test('it will return "undefined" if the to and from are invalid dateMath parsable', () => { + expect( + adjustTimeRange({ + interval: '5m', + to: 'now-invalid', + from: 'now-invalid2', + }) + ).toMatchObject>({ + timeRangeAdjusted: undefined, + duration: undefined, + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts b/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts index 5c99524694a66..e9334379d7b0b 100644 --- a/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts +++ b/x-pack/plugins/security_solution/public/transforms/utils/adjust_timerange.ts @@ -9,11 +9,21 @@ import dateMath from '@elastic/datemath'; import moment, { Duration } from 'moment'; import type { TimerangeInput } from '../../../common/search_strategy'; -export type ParseTimeRange = ( - timeRange: TimerangeInput -) => { timeRangeAdjusted: TimerangeInput | undefined; duration: Duration | undefined }; +export interface TimeRangeAdjusted { + timeRangeAdjusted: TimerangeInput | undefined; + duration: Duration | undefined; +} -export const adjustTimeRange: ParseTimeRange = (timerange) => { +/** + * Adjusts a given timerange by rounding the "from" down by an hour and returning + * the duration between "to" and "from". The duration is typically analyzed to determine + * if the adjustment should be made or not. Although we check "to" and use "to" for duration + * we are careful to still return "to: timerange.to", which is the original input to be careful + * about accidental bugs from trying to over parse or change relative date time ranges. + * @param timerange The timeRange to determine if we adjust or not + * @returns The time input adjustment and a duration + */ +export const adjustTimeRange = (timerange: TimerangeInput): TimeRangeAdjusted => { const from = dateMath.parse(timerange.from); const to = dateMath.parse(timerange.to); if (from == null || to == null) { diff --git a/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.test.ts new file mode 100644 index 0000000000000..2cdb1c4ebc2cd --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.test.ts @@ -0,0 +1,31 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { createIndicesFromPrefix } from './create_indices_from_prefix'; + +/** Get the return type of createIndicesFromPrefix for TypeScript checks against expected */ +type ReturnTypeCreateIndicesFromPrefix = ReturnType; + +describe('create_indices_from_prefix', () => { + test('returns empty array given an empty array', () => { + expect( + createIndicesFromPrefix({ + transformIndices: [], + prefix: 'prefix', + }) + ).toEqual([]); + }); + + test('returns expected prefix given a set of indices', () => { + expect( + createIndicesFromPrefix({ + transformIndices: ['index_1', 'index_2'], + prefix: 'prefix', + }) + ).toEqual(['.estc_prefix_index_1', '.estc_prefix_index_2']); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts b/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts index 1f3f3959f2aa3..de63933ba4475 100644 --- a/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts +++ b/x-pack/plugins/security_solution/public/transforms/utils/create_indices_from_prefix.ts @@ -7,6 +7,13 @@ import { ELASTIC_NAME } from '../../../common/constants'; +/** + * Given a set of input indices and a prefix this will return the elastic name + * concatenated with the prefix. + * @param transformIndices The indices to add the prefix to + * @param prefix The prefix to add along with the elastic name + * @returns The indices with the prefix string + */ export const createIndicesFromPrefix = ({ transformIndices, prefix, diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.test.ts new file mode 100644 index 0000000000000..a58757c261624 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.test.ts @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getSettingsMatch } from './get_settings_match'; +import { getTransformConfigSchemaMock } from './transform_config_schema.mock'; + +/** Get the return type of createIndicesFromPrefix for TypeScript checks against expected */ +type ReturnTypeCreateIndicesFromPrefix = ReturnType; + +describe('get_settings_match', () => { + test('it returns undefined given an empty array of indices', () => { + expect( + getSettingsMatch({ + indices: [], + transformSettings: getTransformConfigSchemaMock(), + }) + ).toEqual(undefined); + }); + + test('it returns a setting given an index pattern that matches', () => { + expect( + getSettingsMatch({ + indices: [ + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + ], + transformSettings: getTransformConfigSchemaMock(), + }) + ).toEqual(getTransformConfigSchemaMock().settings[0]); + }); + + test('it returns a setting given an index pattern that matches even if the indices are different order', () => { + expect( + getSettingsMatch({ + indices: [ + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'auditbeat-*', + 'packetbeat-*', + 'winlogbeat-*', + ], + transformSettings: getTransformConfigSchemaMock(), + }) + ).toEqual(getTransformConfigSchemaMock().settings[0]); + }); + + test('it returns a setting given an index pattern that matches and removes any that have a dash in them meaning to subtract them', () => { + expect( + getSettingsMatch({ + indices: [ + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'auditbeat-*', + 'packetbeat-*', + 'winlogbeat-*', + '-subtract-1', // extra dashed one that should still allow a match + '-subtract-2', // extra dashed one that should still allow a match + ], + transformSettings: getTransformConfigSchemaMock(), + }) + ).toEqual(getTransformConfigSchemaMock().settings[0]); + }); + + test('it returns "undefined" given a set of indices that do not match a setting', () => { + expect( + getSettingsMatch({ + indices: ['endgame-*', 'filebeat-*', 'logs-*', 'auditbeat-*', 'packetbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + }) + ).toEqual(undefined); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts index 1546dddb60ca9..ed7be4a530dcf 100644 --- a/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_settings_match.ts @@ -7,6 +7,12 @@ import { TransformConfigSchema } from '../../../common/transforms/types'; +/** + * Given a transform setting and indices this will return either the particular setting + * that matches the index or it will return undefined if it is not found + * @param indices The indices to check against the transform + * @returns Either the setting if it matches or an undefined if it cannot find one + */ export const getSettingsMatch = ({ indices, transformSettings, diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes.test.ts new file mode 100644 index 0000000000000..5c6e10ae8f7ce --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes.test.ts @@ -0,0 +1,176 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getTransformChanges } from './get_transform_changes'; +import { getTransformConfigSchemaMock } from './transform_config_schema.mock'; +import { + HostsKpiQueries, + HostsQueries, + MatrixHistogramQuery, + MatrixHistogramQueryEntities, + MatrixHistogramType, + NetworkKpiQueries, + NetworkQueries, +} from '../../../common/search_strategy'; + +/** Get the return type of createIndicesFromPrefix for TypeScript checks against expected */ +type ReturnTypeGetTransformChanges = ReturnType; + +describe('get_transform_changes', () => { + describe('kpi transforms', () => { + test('it gets a transform change for kpiHosts', () => { + expect( + getTransformChanges({ + factoryQueryType: HostsKpiQueries.kpiHosts, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsKpiQueries.kpiHostsEntities, + indices: ['.estc_all_host_ent*'], + }); + }); + + test('it gets a transform change for kpiAuthentications', () => { + expect( + getTransformChanges({ + factoryQueryType: HostsKpiQueries.kpiAuthentications, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsKpiQueries.kpiAuthenticationsEntities, + indices: ['.estc_all_user_ent*'], + }); + }); + + test('it gets a transform change for kpiUniqueIps', () => { + expect( + getTransformChanges({ + factoryQueryType: HostsKpiQueries.kpiUniqueIps, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsKpiQueries.kpiUniqueIpsEntities, + indices: ['.estc_all_src_ip_ent*', '.estc_all_dest_ip_ent*'], + }); + }); + }); + + describe('host transforms', () => { + test('it gets a transform change for hosts', () => { + expect( + getTransformChanges({ + factoryQueryType: HostsQueries.hosts, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsQueries.hostsEntities, + indices: ['.estc_all_host_ent*'], + }); + }); + + test('it gets a transform change for authentications', () => { + expect( + getTransformChanges({ + factoryQueryType: HostsQueries.authentications, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsQueries.authenticationsEntities, + indices: ['.estc_all_user_ent*'], + }); + }); + }); + + describe('network transforms', () => { + test('it gets a transform change for topCountries', () => { + expect( + getTransformChanges({ + factoryQueryType: NetworkQueries.topCountries, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkQueries.topCountriesEntities, + indices: ['.estc_all_src_iso_ent*', '.estc_all_dest_iso_ent*'], + }); + }); + + test('it gets a transform change for topNFlow', () => { + expect( + getTransformChanges({ + factoryQueryType: NetworkQueries.topNFlow, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkQueries.topNFlowEntities, + indices: ['.estc_all_src_ip_ent*', '.estc_all_dest_ip_ent*'], + }); + }); + + test('it gets a transform change for dns', () => { + expect( + getTransformChanges({ + factoryQueryType: NetworkKpiQueries.dns, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkKpiQueries.dnsEntities, + indices: ['.estc_all_ip_met*'], + }); + }); + + test('it gets a transform change for networkEvents', () => { + expect( + getTransformChanges({ + factoryQueryType: NetworkKpiQueries.networkEvents, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkKpiQueries.networkEventsEntities, + indices: ['.estc_all_ip_met*'], + }); + }); + + test('it gets a transform change for tlsHandshakes', () => { + expect( + getTransformChanges({ + factoryQueryType: NetworkKpiQueries.tlsHandshakes, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkKpiQueries.tlsHandshakesEntities, + indices: ['.estc_all_ip_met*'], + }); + }); + }); + + describe('matrix transforms', () => { + test('it gets a transform change for authentications', () => { + expect( + getTransformChanges({ + factoryQueryType: MatrixHistogramQuery, + histogramType: MatrixHistogramType.authentications, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + histogramType: MatrixHistogramType.authenticationsEntities, + factoryQueryType: MatrixHistogramQueryEntities, + indices: ['.estc_all_user_met*'], + }); + }); + }); + + describe('unsupported/undefined transforms', () => { + test('it returned unsupported/undefined for firstOrLastSeen since there are not transforms for it', () => { + expect( + getTransformChanges({ + factoryQueryType: HostsQueries.firstOrLastSeen, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual(undefined); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.test.ts new file mode 100644 index 0000000000000..e76c2ee2575ff --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.test.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getTransformChangesForHosts } from './get_transform_changes_for_hosts'; +import { HostsQueries } from '../../../common/search_strategy/security_solution/hosts'; +import { getTransformConfigSchemaMock } from './transform_config_schema.mock'; + +/** Get the return type of getTransformChangesForHosts for TypeScript checks against expected */ +type ReturnTypeGetTransformChangesForHosts = ReturnType; + +describe('get_transform_changes_for_host', () => { + test('it gets a transform change for hosts', () => { + expect( + getTransformChangesForHosts({ + factoryQueryType: HostsQueries.hosts, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsQueries.hostsEntities, + indices: ['.estc_all_host_ent*'], + }); + }); + + test('it gets a transform change for authentications', () => { + expect( + getTransformChangesForHosts({ + factoryQueryType: HostsQueries.authentications, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsQueries.authenticationsEntities, + indices: ['.estc_all_user_ent*'], + }); + }); + + test('it returns an "undefined" for another value', () => { + expect( + getTransformChangesForHosts({ + factoryQueryType: HostsQueries.firstOrLastSeen, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual(undefined); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.ts index fef884dd6761f..95a5e04bd9e51 100644 --- a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.ts +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_hosts.ts @@ -9,6 +9,13 @@ import { HostsQueries } from '../../../common/search_strategy'; import { createIndicesFromPrefix } from './create_indices_from_prefix'; import { GetTransformChanges } from './types'; +/** + * Given a factory query type this will return the transform changes such as the transform indices if it matches + * the correct type, otherwise it will return "undefined" + * @param factoryQueryType The query type to check if we have a transform for it and are capable of rendering one or not + * @param settings The settings configuration to get the prefix from + * @returns The transform type if we have one, otherwise undefined + */ export const getTransformChangesForHosts: GetTransformChanges = ({ factoryQueryType, settings, diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.test.ts new file mode 100644 index 0000000000000..561874930feae --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.test.ts @@ -0,0 +1,61 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getTransformChangesForKpi } from './get_transform_changes_for_kpi'; +import { HostsKpiQueries } from '../../../common/search_strategy'; +import { HostsQueries } from '../../../common/search_strategy/security_solution/hosts'; +import { getTransformConfigSchemaMock } from './transform_config_schema.mock'; + +/** Get the return type of getTransformChangesForKpi for TypeScript checks against expected */ +type ReturnTypeGetTransformChangesForKpi = ReturnType; + +describe('get_transform_changes_for_kpi', () => { + test('it gets a transform change for kpiHosts', () => { + expect( + getTransformChangesForKpi({ + factoryQueryType: HostsKpiQueries.kpiHosts, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsKpiQueries.kpiHostsEntities, + indices: ['.estc_all_host_ent*'], + }); + }); + + test('it gets a transform change for kpiAuthentications', () => { + expect( + getTransformChangesForKpi({ + factoryQueryType: HostsKpiQueries.kpiAuthentications, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsKpiQueries.kpiAuthenticationsEntities, + indices: ['.estc_all_user_ent*'], + }); + }); + + test('it gets a transform change for kpiUniqueIps', () => { + expect( + getTransformChangesForKpi({ + factoryQueryType: HostsKpiQueries.kpiUniqueIps, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: HostsKpiQueries.kpiUniqueIpsEntities, + indices: ['.estc_all_src_ip_ent*', '.estc_all_dest_ip_ent*'], + }); + }); + + test('it returns an "undefined" for another value', () => { + expect( + getTransformChangesForKpi({ + factoryQueryType: HostsQueries.firstOrLastSeen, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual(undefined); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.ts index 620e794dd0a77..800d3189e0c7a 100644 --- a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.ts +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_kpi.ts @@ -8,6 +8,13 @@ import { HostsKpiQueries } from '../../../common/search_strategy'; import { createIndicesFromPrefix } from './create_indices_from_prefix'; import { GetTransformChanges } from './types'; +/** + * Given a factory query type this will return the transform changes such as the transform indices if it matches + * the correct type, otherwise it will return "undefined" + * @param factoryQueryType The query type to check if we have a transform for it and are capable of rendering one or not + * @param settings The settings configuration to get the prefix from + * @returns The transform type if we have one, otherwise undefined + */ export const getTransformChangesForKpi: GetTransformChanges = ({ factoryQueryType, settings }) => { switch (factoryQueryType) { case HostsKpiQueries.kpiHosts: { diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.test.ts new file mode 100644 index 0000000000000..539fb960e4411 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.test.ts @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + MatrixHistogramType, + MatrixHistogramQuery, + MatrixHistogramQueryEntities, +} from '../../../common/search_strategy'; +import { getTransformChangesForMatrixHistogram } from './get_transform_changes_for_matrix_histogram'; +import { HostsQueries } from '../../../common/search_strategy/security_solution/hosts'; +import { getTransformConfigSchemaMock } from './transform_config_schema.mock'; + +/** Get the return type of getTransformChangesForMatrixHistogram for TypeScript checks against expected */ +type ReturnTypeGetTransformChangesForMatrixHistogram = ReturnType< + typeof getTransformChangesForMatrixHistogram +>; + +describe('get_transform_changes_for_matrix_histogram', () => { + test('it gets a transform change for authentications', () => { + expect( + getTransformChangesForMatrixHistogram({ + factoryQueryType: MatrixHistogramQuery, + histogramType: MatrixHistogramType.authentications, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + histogramType: MatrixHistogramType.authenticationsEntities, + factoryQueryType: MatrixHistogramQueryEntities, + indices: ['.estc_all_user_met*'], + }); + }); + + test('it returns an "undefined" for another value', () => { + expect( + getTransformChangesForMatrixHistogram({ + factoryQueryType: HostsQueries.firstOrLastSeen, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual(undefined); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.ts index fca29b2e97d87..36e8c2203200b 100644 --- a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.ts +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_matrix_histogram.ts @@ -13,6 +13,14 @@ import { import { createIndicesFromPrefix } from './create_indices_from_prefix'; import { GetTransformChanges } from './types'; +/** + * Given a factory query type this will return the transform changes such as the transform indices if it matches + * the correct type, otherwise it will return "undefined" + * @param factoryQueryType The query type to check if we have a transform for it and are capable of rendering one or not + * @param histogramType The histogram type to check if we have a transform for it and are capable fo rendering one or not + * @param settings The settings configuration to get the prefix from + * @returns The transform type if we have one, otherwise undefined + */ export const getTransformChangesForMatrixHistogram: GetTransformChanges = ({ factoryQueryType, settings, diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.test.ts new file mode 100644 index 0000000000000..ffac8853f6e67 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.test.ts @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { getTransformChangesForNetwork } from './get_transform_changes_for_network'; +import { NetworkKpiQueries, NetworkQueries } from '../../../common/search_strategy'; +import { HostsQueries } from '../../../common/search_strategy/security_solution/hosts'; +import { getTransformConfigSchemaMock } from './transform_config_schema.mock'; + +/** Get the return type of getTransformChangesForNetwork for TypeScript checks against expected */ +type ReturnTypeGetTransformChangesForNetwork = ReturnType; + +describe('get_transform_changes_for_network', () => { + test('it gets a transform change for topCountries', () => { + expect( + getTransformChangesForNetwork({ + factoryQueryType: NetworkQueries.topCountries, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkQueries.topCountriesEntities, + indices: ['.estc_all_src_iso_ent*', '.estc_all_dest_iso_ent*'], + }); + }); + + test('it gets a transform change for topNFlow', () => { + expect( + getTransformChangesForNetwork({ + factoryQueryType: NetworkQueries.topNFlow, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkQueries.topNFlowEntities, + indices: ['.estc_all_src_ip_ent*', '.estc_all_dest_ip_ent*'], + }); + }); + + test('it gets a transform change for dns', () => { + expect( + getTransformChangesForNetwork({ + factoryQueryType: NetworkKpiQueries.dns, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkKpiQueries.dnsEntities, + indices: ['.estc_all_ip_met*'], + }); + }); + + test('it gets a transform change for networkEvents', () => { + expect( + getTransformChangesForNetwork({ + factoryQueryType: NetworkKpiQueries.networkEvents, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkKpiQueries.networkEventsEntities, + indices: ['.estc_all_ip_met*'], + }); + }); + + test('it gets a transform change for tlsHandshakes', () => { + expect( + getTransformChangesForNetwork({ + factoryQueryType: NetworkKpiQueries.tlsHandshakes, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkKpiQueries.tlsHandshakesEntities, + indices: ['.estc_all_ip_met*'], + }); + }); + + test('it gets a transform change for uniquePrivateIps', () => { + expect( + getTransformChangesForNetwork({ + factoryQueryType: NetworkKpiQueries.uniquePrivateIps, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual({ + factoryQueryType: NetworkKpiQueries.uniquePrivateIpsEntities, + indices: ['.estc_all_src_ip_ent*', '.estc_all_dest_ip_ent*'], + }); + }); + + test('it returns an "undefined" for another value', () => { + expect( + getTransformChangesForNetwork({ + factoryQueryType: HostsQueries.firstOrLastSeen, + settings: getTransformConfigSchemaMock().settings[0], + }) + ).toEqual(undefined); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.ts index 1b1ef815f82bc..a1ec0783235f8 100644 --- a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.ts +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_for_network.ts @@ -9,6 +9,13 @@ import { NetworkKpiQueries, NetworkQueries } from '../../../common/search_strate import { createIndicesFromPrefix } from './create_indices_from_prefix'; import { GetTransformChanges } from './types'; +/** + * Given a factory query type this will return the transform changes such as the transform indices if it matches + * the correct type, otherwise it will return "undefined" + * @param factoryQueryType The query type to check if we have a transform for it and are capable of rendering one or not + * @param settings The settings configuration to get the prefix from + * @returns The transform type if we have one, otherwise undefined + */ export const getTransformChangesForNetwork: GetTransformChanges = ({ factoryQueryType, settings, diff --git a/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_if_they_exist.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_if_they_exist.test.ts new file mode 100644 index 0000000000000..e5d290500499e --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/get_transform_changes_if_they_exist.test.ts @@ -0,0 +1,301 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { getTransformChangesIfTheyExist } from './get_transform_changes_if_they_exist'; +import { HostsKpiQueries, HostsQueries } from '../../../common/search_strategy'; +import moment from 'moment'; +import { getTransformConfigSchemaMock } from './transform_config_schema.mock'; + +/** Get the return type of createIndicesFromPrefix for TypeScript checks against expected */ +type ReturnTypeGetTransformChangesIfTheyExist = ReturnType; + +describe('get_transform_changes_if_they_exist', () => { + beforeEach(() => { + // Adds extra switch to suppress deprecation warnings that moment does not expose in TypeScript + (moment as typeof moment & { + suppressDeprecationWarnings: boolean; + }).suppressDeprecationWarnings = true; + }); + + afterEach(() => { + // Adds extra switch to suppress deprecation warnings that moment does not expose in TypeScript + (moment as typeof moment & { + suppressDeprecationWarnings: boolean; + }).suppressDeprecationWarnings = false; + }); + + describe('transformSettings enabled conditional logic', () => { + test('returns transformed settings if our settings is enabled', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['auditbeat-*'], + transformSettings: { ...getTransformConfigSchemaMock(), enabled: true }, // sets enabled to true + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['.estc_all_user_ent*'], + factoryQueryType: HostsQueries.authenticationsEntities, + }); + }); + + test('returns regular settings if our settings is disabled', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['auditbeat-*'], + transformSettings: { ...getTransformConfigSchemaMock(), enabled: false }, // sets enabled to false + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['auditbeat-*'], + factoryQueryType: HostsQueries.authentications, + }); + }); + }); + + describe('filter query compatibility conditional logic', () => { + test('returns regular settings if filter is set to something other than match_all', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['auditbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: { + bool: { + must: [], + filter: [{ match_none: {} }], // match_none shouldn't return transform + should: [], + must_not: [], + }, + }, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['auditbeat-*'], + factoryQueryType: HostsQueries.authentications, + }); + }); + + test('returns transformed settings if filter is set to something such as match_all', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['auditbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: { + bool: { + must: [], + filter: [{ match_all: {} }], // match_all should return transform + should: [], + must_not: [], + }, + }, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['.estc_all_user_ent*'], + factoryQueryType: HostsQueries.authenticationsEntities, + }); + }); + + test('returns transformed settings if filter is set to undefined', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['auditbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: undefined, // undefined should return transform + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['.estc_all_user_ent*'], + factoryQueryType: HostsQueries.authenticationsEntities, + }); + }); + }); + + describe('timerange adjustments conditional logic', () => { + test('returns regular settings if timerange is less than an hour', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['auditbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', // Less than hour + from: '2021-07-06T23:39:38.643Z', // Less than hour + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['auditbeat-*'], + factoryQueryType: HostsQueries.authentications, + }); + }); + + test('returns regular settings if timerange is invalid', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['auditbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: 'now-invalid', // invalid to + from: 'now-invalid2', // invalid from + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['auditbeat-*'], + factoryQueryType: HostsQueries.authentications, + }); + }); + + test('returns transformed settings if timerange is greater than an hour', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['auditbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', // Greater than an hour + from: '2021-07-06T20:49:38.643Z', // Greater than an hour + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['.estc_all_user_ent*'], + factoryQueryType: HostsQueries.authenticationsEntities, + }); + }); + }); + + describe('settings match conditional logic', () => { + test('it returns regular settings if settings do not match', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: ['should-not-match-*'], // index doesn't match anything + transformSettings: getTransformConfigSchemaMock(), + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['should-not-match-*'], + factoryQueryType: HostsQueries.authentications, + }); + }); + + test('it returns transformed settings if settings do match', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.authentications, + indices: [ + 'auditbeat-*', + 'endgame-*', + 'filebeat-*', + 'logs-*', + 'packetbeat-*', + 'winlogbeat-*', + '-subtract-something', + ], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['.estc_all_user_ent*'], + factoryQueryType: HostsQueries.authenticationsEntities, + }); + }); + }); + + describe('transform changes conditional logic', () => { + test('it returns regular settings if it does not match a transform factory type', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsQueries.firstOrLastSeen, // query type not used for any transforms yet + indices: ['auditbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['auditbeat-*'], + factoryQueryType: HostsQueries.firstOrLastSeen, + }); + }); + + test('it returns transformed settings if it does match a transform factory type', () => { + expect( + getTransformChangesIfTheyExist({ + factoryQueryType: HostsKpiQueries.kpiHosts, // valid kpiHosts for a transform + indices: ['auditbeat-*'], + transformSettings: getTransformConfigSchemaMock(), + filterQuery: undefined, + histogramType: undefined, + timerange: { + to: '2021-07-06T23:49:38.643Z', + from: '2021-07-06T20:49:38.643Z', + interval: '5m', + }, + }) + ).toMatchObject>({ + indices: ['.estc_all_host_ent*'], + factoryQueryType: HostsKpiQueries.kpiHostsEntities, + }); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.test.ts b/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.test.ts new file mode 100644 index 0000000000000..74dd1373a4d39 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.test.ts @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { isFilterQueryCompatible } from './is_filter_query_compatible'; + +describe('is_filter_query_compatible', () => { + test('returns true if given an undefined', () => { + expect(isFilterQueryCompatible(undefined)).toEqual(true); + }); + + test('returns "true" if given a match all object', () => { + expect( + isFilterQueryCompatible({ + bool: { + must: [], + filter: [{ match_all: {} }], + should: [], + must_not: [], + }, + }) + ).toEqual(true); + }); + + test('returns "false" if given a match all object with something inside of it such as match_none', () => { + expect( + isFilterQueryCompatible({ + bool: { + must: [], + filter: [{ match_none: {} }], + should: [], + must_not: [], + }, + }) + ).toEqual(false); + }); + + test('returns "true" if given empty array for a filter', () => { + expect( + isFilterQueryCompatible({ + bool: { + must: [], + filter: [], + should: [], + must_not: [], + }, + }) + ).toEqual(true); + }); + + test('returns "true" if given match all object as a string', () => { + expect( + isFilterQueryCompatible( + JSON.stringify({ + bool: { + must: [], + filter: [], + should: [], + must_not: [], + }, + }) + ) + ).toEqual(true); + }); + + test('returns "true" if given empty array for a filter as a string', () => { + expect( + isFilterQueryCompatible( + JSON.stringify({ + bool: { + must: [], + filter: [], + should: [], + must_not: [], + }, + }) + ) + ).toEqual(true); + }); + + test('returns "false" if given an invalid string', () => { + expect(isFilterQueryCompatible('invalid string')).toEqual(false); + }); +}); diff --git a/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.ts b/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.ts index 31c264bce3cd3..2f2c9ee7f2deb 100644 --- a/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.ts +++ b/x-pack/plugins/security_solution/public/transforms/utils/is_filter_query_compatible.ts @@ -5,17 +5,50 @@ * 2.0. */ +import { isEqual } from 'lodash/fp'; import { ESQuery } from '../../../common/typed_json'; -export const isFilterQueryCompatible = (filterQuery: ESQuery | string | undefined) => { +/** + * Array of query compatible objects which are at the moment all + * simple empty or match all based objects + */ +const queryCompatibleStrings: ESQuery[] = [ + { + bool: { + must: [], + filter: [{ match_all: {} }], + should: [], + must_not: [], + }, + }, + { + bool: { + must: [], + filter: [], + should: [], + must_not: [], + }, + }, +]; + +/** + * Returns true if the filter query matches against one of the compatible strings, otherwise + * false. Right now we only check if the filter query is empty, or a match all in order to activate + * the transform. + * @param filterQuery The filterQuery to check against and return true if it matches, otherwise false. + * @returns true if the filter is compatible, otherwise false. + */ +export const isFilterQueryCompatible = (filterQuery: ESQuery | string | undefined): boolean => { if (filterQuery === undefined) { return true; } else if (typeof filterQuery === 'string') { - return ( - filterQuery === '{"bool":{"must":[],"filter":[{"match_all":{}}],"should":[],"must_not":[]}}' - ); + try { + const filterQueryObject = JSON.parse(filterQuery); + return queryCompatibleStrings.some((entry) => isEqual(entry, filterQueryObject)); + } catch (error) { + return false; + } } else { - // TODO: Can we check here and return if it matches a string or other signature? - return false; + return queryCompatibleStrings.some((entry) => isEqual(entry, filterQuery)); } }; diff --git a/x-pack/plugins/security_solution/public/transforms/utils/transform_config_schema.mock.ts b/x-pack/plugins/security_solution/public/transforms/utils/transform_config_schema.mock.ts new file mode 100644 index 0000000000000..ef3d4bfe6f007 --- /dev/null +++ b/x-pack/plugins/security_solution/public/transforms/utils/transform_config_schema.mock.ts @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +import { TransformConfigSchema } from '../../../common/transforms/types'; + +/** + * Mock for the TransformConfigSchema. + * @returns A transform config schema mock + */ +export const getTransformConfigSchemaMock = (): TransformConfigSchema => ({ + enabled: true, + auto_start: true, + auto_create: true, + query: { + range: { + '@timestamp': { + gte: 'now-1d/d', + format: 'strict_date_optional_time', + }, + }, + }, + retention_policy: { + time: { + field: '@timestamp', + max_age: '1w', + }, + }, + max_page_search_size: 5000, + settings: [ + { + prefix: 'all', + indices: ['auditbeat-*', 'endgame-*', 'filebeat-*', 'logs-*', 'packetbeat-*', 'winlogbeat-*'], + data_sources: [ + ['auditbeat-*', 'endgame-*', 'filebeat-*', 'logs-*', 'packetbeat-*', 'winlogbeat-*'], + ['auditbeat-*', 'filebeat-*', 'logs-*', 'winlogbeat-*'], + ['auditbeat-*'], + ], + }, + ], +});