From ab42ca2c64f44369cdcf873c84a7ace8e7e2d68d Mon Sep 17 00:00:00 2001 From: Nathan Reese Date: Tue, 6 Oct 2020 15:48:50 -0600 Subject: [PATCH 01/83] [Maps] fix use correct mount-context (#79688) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../plugins/maps/public/lazy_load_bundle/index.ts | 4 ++-- x-pack/plugins/maps/public/plugin.ts | 5 +++-- x-pack/plugins/maps/public/routing/maps_router.tsx | 13 ++++++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/x-pack/plugins/maps/public/lazy_load_bundle/index.ts b/x-pack/plugins/maps/public/lazy_load_bundle/index.ts index 9bced75b613d72..9fbe090633747e 100644 --- a/x-pack/plugins/maps/public/lazy_load_bundle/index.ts +++ b/x-pack/plugins/maps/public/lazy_load_bundle/index.ts @@ -7,7 +7,7 @@ import { AnyAction } from 'redux'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { IndexPatternsContract } from 'src/plugins/data/public/index_patterns'; -import { AppMountContext, AppMountParameters } from 'kibana/public'; +import { AppMountParameters } from 'kibana/public'; import { IndexPattern } from 'src/plugins/data/public'; import { Embeddable, IContainer } from '../../../../../src/plugins/embeddable/public'; import { LayerDescriptor } from '../../common/descriptor_types'; @@ -40,7 +40,7 @@ interface LazyLoadedMapModules { initialLayers?: LayerDescriptor[] ) => LayerDescriptor[]; mergeInputWithSavedMap: any; - renderApp: (context: AppMountContext, params: AppMountParameters) => Promise<() => void>; + renderApp: (params: AppMountParameters) => Promise<() => void>; createSecurityLayerDescriptors: ( indexPatternId: string, indexPatternTitle: string diff --git a/x-pack/plugins/maps/public/plugin.ts b/x-pack/plugins/maps/public/plugin.ts index 5b79863d0dd97b..a2b629bdd4989e 100644 --- a/x-pack/plugins/maps/public/plugin.ts +++ b/x-pack/plugins/maps/public/plugin.ts @@ -9,6 +9,7 @@ import { UiActionsStart } from 'src/plugins/ui_actions/public'; import { NavigationPublicPluginStart } from 'src/plugins/navigation/public'; import { Start as InspectorStartContract } from 'src/plugins/inspector/public'; import { + AppMountParameters, CoreSetup, CoreStart, Plugin, @@ -131,9 +132,9 @@ export class MapsPlugin icon: `plugins/${APP_ID}/icon.svg`, euiIconType: APP_ICON_SOLUTION, category: DEFAULT_APP_CATEGORIES.kibana, - async mount(context, params) { + async mount(params: AppMountParameters) { const { renderApp } = await lazyLoadMapModules(); - return renderApp(context, params); + return renderApp(params); }, }); } diff --git a/x-pack/plugins/maps/public/routing/maps_router.tsx b/x-pack/plugins/maps/public/routing/maps_router.tsx index a28c293a2f32f3..d7e6e6e0799530 100644 --- a/x-pack/plugins/maps/public/routing/maps_router.tsx +++ b/x-pack/plugins/maps/public/routing/maps_router.tsx @@ -9,7 +9,7 @@ import { render, unmountComponentAtNode } from 'react-dom'; import { Router, Switch, Route, Redirect, RouteComponentProps } from 'react-router-dom'; import { i18n } from '@kbn/i18n'; import { Provider } from 'react-redux'; -import { AppMountContext, AppMountParameters } from 'kibana/public'; +import { AppMountParameters } from 'kibana/public'; import { getCoreChrome, getCoreI18n, @@ -29,10 +29,13 @@ import { LoadMapAndRender } from './routes/maps_app/load_map_and_render'; export let goToSpecifiedPath: (path: string) => void; export let kbnUrlStateStorage: IKbnUrlStateStorage; -export async function renderApp( - context: AppMountContext, - { appBasePath, element, history, onAppLeave, setHeaderActionMenu }: AppMountParameters -) { +export async function renderApp({ + appBasePath, + element, + history, + onAppLeave, + setHeaderActionMenu, +}: AppMountParameters) { goToSpecifiedPath = (path) => history.push(path); kbnUrlStateStorage = createKbnUrlStateStorage({ useHash: false, From 92ce8f3040991bd4473a93e9ee89b1f46580b87a Mon Sep 17 00:00:00 2001 From: Tyler Smalley Date: Tue, 6 Oct 2020 14:54:30 -0700 Subject: [PATCH 02/83] skips test failing promotion (#79777) Signed-off-by: Tyler Smalley --- .../apps/ml/data_frame_analytics/outlier_detection_creation.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts index b5b0f4c94f2621..e0bd88a6ab6426 100644 --- a/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts +++ b/x-pack/test/functional/apps/ml/data_frame_analytics/outlier_detection_creation.ts @@ -11,7 +11,8 @@ export default function ({ getService }: FtrProviderContext) { const ml = getService('ml'); const editedDescription = 'Edited description'; - describe('outlier detection creation', function () { + // https://github.com/elastic/kibana/issues/79777 + describe.skip('outlier detection creation', function () { before(async () => { await esArchiver.loadIfNeeded('ml/ihp_outlier'); await ml.testResources.createIndexPatternIfNeeded('ft_ihp_outlier', '@timestamp'); From 68130dfd87b7c6c804f610d28909d560ada126ec Mon Sep 17 00:00:00 2001 From: Nathan L Smith Date: Tue, 6 Oct 2020 16:59:56 -0500 Subject: [PATCH 03/83] Re-enable canvas storyshots (#79750) * Only throw the error about building the DLL if we're not running in Jest * Update existing storyshots * Add Jest mock for Datasource story All tests look to be passing and `yarn storybook canvas` works. I clicked through all the stories and it doesn't look like anything is broken. --- .../datasource_component.stories.storyshot | 129 ++++++++++++++++++ .../simple_template.stories.storyshot | 2 - x-pack/plugins/canvas/storybook/main.ts | 7 +- .../canvas/storybook/storyshots.test.tsx | 22 +-- 4 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 x-pack/plugins/canvas/public/components/datasource/__stories__/__snapshots__/datasource_component.stories.storyshot diff --git a/x-pack/plugins/canvas/public/components/datasource/__stories__/__snapshots__/datasource_component.stories.storyshot b/x-pack/plugins/canvas/public/components/datasource/__stories__/__snapshots__/datasource_component.stories.storyshot new file mode 100644 index 00000000000000..373c147c2a5b80 --- /dev/null +++ b/x-pack/plugins/canvas/public/components/datasource/__stories__/__snapshots__/datasource_component.stories.storyshot @@ -0,0 +1,129 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Storyshots components/datasource/DatasourceComponent datasource with expression arguments 1`] = ` +
+ +
+
+
+

+ The datasource has an argument controlled by an expression. Use the expression editor to modify the datasource. +

+
+
+
+`; + +exports[`Storyshots components/datasource/DatasourceComponent simple datasource 1`] = ` +
+ +
+
+
+
+
+ +
+
+ +
+
+
+`; diff --git a/x-pack/plugins/canvas/public/expression_types/arg_types/container_style/__stories__/__snapshots__/simple_template.stories.storyshot b/x-pack/plugins/canvas/public/expression_types/arg_types/container_style/__stories__/__snapshots__/simple_template.stories.storyshot index 495bf5262476ce..9eeffde84ffaca 100644 --- a/x-pack/plugins/canvas/public/expression_types/arg_types/container_style/__stories__/__snapshots__/simple_template.stories.storyshot +++ b/x-pack/plugins/canvas/public/expression_types/arg_types/container_style/__stories__/__snapshots__/simple_template.stories.storyshot @@ -28,7 +28,6 @@ exports[`Storyshots arguments/ContainerStyle simple 1`] = ` >
({ + getDefaultIndex: () => Promise.resolve('test index'), +})); + addSerializer(styleSheetSerializer); // Initialize Storyshots and build the Jest Snapshots -// Commenting this out until after #75357 is merged and Jest gets updated. -// initStoryshots({ -// configPath: path.resolve(__dirname, './../storybook'), -// test: multiSnapshotWithOptions({}), -// // Don't snapshot tests that start with 'redux' -// storyNameRegex: /^((?!.*?redux).)*$/, -// }); - -test.todo('Storyshots'); +initStoryshots({ + configPath: path.resolve(__dirname, './../storybook'), + test: multiSnapshotWithOptions({}), + // Don't snapshot tests that start with 'redux' + storyNameRegex: /^((?!.*?redux).)*$/, +}); From 13b8aed11d0461e5e1c684964fb3a30555a87edd Mon Sep 17 00:00:00 2001 From: Joe Portner <5295965+jportner@users.noreply.github.com> Date: Tue, 6 Oct 2020 18:00:23 -0400 Subject: [PATCH 04/83] [Docs] Update developer docs for create/bulkCreate `initialNamespaces` (#79769) Co-authored-by: gchaps <33642766+gchaps@users.noreply.github.com> --- docs/api/saved-objects/bulk_create.asciidoc | 5 +++-- docs/api/saved-objects/create.asciidoc | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/api/saved-objects/bulk_create.asciidoc b/docs/api/saved-objects/bulk_create.asciidoc index e77559f5d86448..267ab3891d7000 100644 --- a/docs/api/saved-objects/bulk_create.asciidoc +++ b/docs/api/saved-objects/bulk_create.asciidoc @@ -42,8 +42,9 @@ experimental[] Create multiple {kib} saved objects. (Optional, array) Objects with `name`, `id`, and `type` properties that describe the other saved objects in the referenced object. To refer to the other saved object, use `name` in the attributes. Never use `id` to refer to the other saved object. `id` can be automatically updated during migrations, import, or export. `initialNamespaces`:: - (Optional, string array) Identifiers for the <> in which this object should be created. If this is not provided, the - object will be created in the current space. + (Optional, string array) Identifiers for the <> in which this object is created. If this is provided, the + object is created only in the explicitly defined spaces. If this is not provided, the object is created in the current space + (default behavior). `version`:: (Optional, number) Specifies the version. diff --git a/docs/api/saved-objects/create.asciidoc b/docs/api/saved-objects/create.asciidoc index fac4f2bf109fab..50809a1bd5d4e2 100644 --- a/docs/api/saved-objects/create.asciidoc +++ b/docs/api/saved-objects/create.asciidoc @@ -47,8 +47,9 @@ any data that you send to the API is properly formed. (Optional, array) Objects with `name`, `id`, and `type` properties that describe the other saved objects that this object references. Use `name` in attributes to refer to the other saved object, but never the `id`, which can update automatically during migrations or import/export. `initialNamespaces`:: - (Optional, string array) Identifiers for the <> in which this object should be created. If this is not provided, the - object will be created in the current space. + (Optional, string array) Identifiers for the <> in which this object is created. If this is provided, the + object is created only in the explicitly defined spaces. If this is not provided, the object is created in the current space + (default behavior). [[saved-objects-api-create-request-codes]] ==== Response code From 302004df47293b1148392fb4dcdfce5c6312a155 Mon Sep 17 00:00:00 2001 From: John Schulz Date: Tue, 6 Oct 2020 18:02:44 -0400 Subject: [PATCH 05/83] [Ingest Manager] Configure Elasticsearch output with YAML in global output settings (#79019) ## Summary YAML entered on the Settings page will be added to each Elastic Agent policy ### Settings form Screen Shot 2020-10-05 at 9 29 27 AMScreen Shot 2020-10-05 at 9 29 58 AM ### Policy screen Screen Shot 2020-10-05 at 9 30 38 AM ### Input Validation
Fails if the value cannot be parsed as YAML, but there are no restricted keys or other guidance Screen Shot 2020-10-05 at 9 32 39 AM
### Open questions 1. Is "Additional YAML Configuration" ok for the form? * Will use "Elasticsearch output configuration" 1. Alternatives to [`additional_yaml_config`](https://github.com/elastic/kibana/pull/79019/files#diff-d2cc0ddf9161efb6898baca37350c720) for the new saved object key * No comments on the name but will move it to the outputs saved object ### Checklist - [x] Any text added follows [EUI's writing guidelines](https://elastic.github.io/eui/#/guidelines/writing), uses sentence case text and includes [i18n support](https://github.com/elastic/kibana/blob/master/packages/kbn-i18n/README.md) --- .../common/types/models/output.ts | 1 + .../common/types/rest_spec/output.ts | 2 + .../components/settings_flyout.tsx | 53 ++++++++++++++++++- .../components/agent_policy_yaml_flyout.tsx | 4 +- .../server/saved_objects/index.ts | 2 + .../server/services/agent_policy.ts | 14 +++-- .../server/types/models/output.ts | 1 + .../server/types/rest_spec/output.ts | 2 + .../server/types/rest_spec/settings.ts | 1 + 9 files changed, 73 insertions(+), 7 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/models/output.ts b/x-pack/plugins/ingest_manager/common/types/models/output.ts index f3e76cd167b3f9..a02c8842709191 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/output.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/output.ts @@ -18,6 +18,7 @@ export interface NewOutput { fleet_enroll_username?: string; fleet_enroll_password?: string; config?: Record; + config_yaml?: string; } export type OutputSOAttributes = NewOutput; diff --git a/x-pack/plugins/ingest_manager/common/types/rest_spec/output.ts b/x-pack/plugins/ingest_manager/common/types/rest_spec/output.ts index 87e8a0977e3ba7..bee4814e31b9e2 100644 --- a/x-pack/plugins/ingest_manager/common/types/rest_spec/output.ts +++ b/x-pack/plugins/ingest_manager/common/types/rest_spec/output.ts @@ -22,6 +22,8 @@ export interface PutOutputRequest { body: { hosts?: string[]; ca_sha256?: string; + config?: Record; + config_yaml?: string; }; } diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/settings_flyout.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/settings_flyout.tsx index e0d843ad773b86..9e66fc7b37b577 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/settings_flyout.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/components/settings_flyout.tsx @@ -20,10 +20,12 @@ import { EuiFormRow, EuiRadioGroup, EuiComboBox, + EuiCodeEditor, } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiText } from '@elastic/eui'; -import { useComboInput, useCore, useGetSettings, sendPutSettings } from '../hooks'; +import { safeLoad } from 'js-yaml'; +import { useComboInput, useCore, useGetSettings, useInput, sendPutSettings } from '../hooks'; import { useGetOutputs, sendPutOutput } from '../hooks/use_request/outputs'; import { isDiffPathProtocol } from '../../../../common/'; @@ -69,10 +71,27 @@ function useSettingsForm(outputId: string | undefined, onSuccess: () => void) { } }); + const additionalYamlConfigInput = useInput('', (value) => { + try { + safeLoad(value); + return; + } catch (error) { + return [ + i18n.translate('xpack.ingestManager.settings.invalidYamlFormatErrorMessage', { + defaultMessage: 'Invalid YAML: {reason}', + values: { reason: error.message }, + }), + ]; + } + }); return { isLoading, onSubmit: async () => { - if (!kibanaUrlsInput.validate() || !elasticsearchUrlInput.validate()) { + if ( + !kibanaUrlsInput.validate() || + !elasticsearchUrlInput.validate() || + !additionalYamlConfigInput.validate() + ) { return; } @@ -83,6 +102,7 @@ function useSettingsForm(outputId: string | undefined, onSuccess: () => void) { } const outputResponse = await sendPutOutput(outputId, { hosts: elasticsearchUrlInput.value, + config_yaml: additionalYamlConfigInput.value, }); if (outputResponse.error) { throw outputResponse.error; @@ -110,6 +130,7 @@ function useSettingsForm(outputId: string | undefined, onSuccess: () => void) { inputs: { kibanaUrls: kibanaUrlsInput, elasticsearchUrl: elasticsearchUrlInput, + additionalYamlConfig: additionalYamlConfigInput, }, }; } @@ -124,6 +145,10 @@ export const SettingFlyout: React.FunctionComponent = ({ onClose }) => { useEffect(() => { if (output) { inputs.elasticsearchUrl.setValue(output.hosts || []); + inputs.additionalYamlConfig.setValue( + output.config_yaml || + `# YAML settings here will be added to the Elasticsearch output section of each policy` + ); } // eslint-disable-next-line react-hooks/exhaustive-deps }, [output]); @@ -247,6 +272,30 @@ export const SettingFlyout: React.FunctionComponent = ({ onClose }) => { + + + + + + ); diff --git a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/components/agent_policy_yaml_flyout.tsx b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/components/agent_policy_yaml_flyout.tsx index 5d485a6e210865..fefb427df5ea69 100644 --- a/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/components/agent_policy_yaml_flyout.tsx +++ b/x-pack/plugins/ingest_manager/public/applications/ingest_manager/sections/agent_policy/components/agent_policy_yaml_flyout.tsx @@ -51,7 +51,9 @@ export const AgentPolicyYamlFlyout = memo<{ policyId: string; onClose: () => voi {error.message} ) : ( - + // Property 'whiteSpace' does not exist on type 'IntrinsicAttributes & CommonProps & OwnProps & HTMLAttributes & { children?: ReactNode; }'. + // @ts-expect-error linter complains whiteSpace isn't available but docs show it on EuiCodeBlockImpl + {fullAgentPolicyToYaml(yamlData!.item)} ); diff --git a/x-pack/plugins/ingest_manager/server/saved_objects/index.ts b/x-pack/plugins/ingest_manager/server/saved_objects/index.ts index 5fc301ceab20f2..8f1ece923f126f 100644 --- a/x-pack/plugins/ingest_manager/server/saved_objects/index.ts +++ b/x-pack/plugins/ingest_manager/server/saved_objects/index.ts @@ -207,6 +207,7 @@ const getSavedObjectTypes = ( fleet_enroll_username: { type: 'binary' }, fleet_enroll_password: { type: 'binary' }, config: { type: 'flattened' }, + config_yaml: { type: 'text' }, }, }, }, @@ -347,6 +348,7 @@ export function registerEncryptedSavedObjects( 'hosts', 'ca_sha256', 'config', + 'config_yaml', ]), }); encryptedSavedObjects.registerType({ diff --git a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts index db1b3f1b9eb9eb..37d5d49fcbfeff 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts @@ -4,6 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ import { uniq } from 'lodash'; +import { safeLoad } from 'js-yaml'; import { SavedObjectsClientContract, SavedObjectsBulkUpdateResponse } from 'src/core/server'; import { AuthenticatedUser } from '../../../security/server'; import { @@ -20,8 +21,12 @@ import { AgentPolicyStatus, ListWithKuery, } from '../types'; +import { + DeleteAgentPolicyResponse, + Settings, + storedPackagePoliciesToAgentInputs, +} from '../../common'; import { AgentPolicyNameExistsError } from '../errors'; -import { DeleteAgentPolicyResponse, storedPackagePoliciesToAgentInputs } from '../../common'; import { createAgentPolicyAction, listAgents } from './agents'; import { packagePolicyService } from './package_policy'; import { outputService } from './output'; @@ -484,13 +489,14 @@ class AgentPolicyService { // TEMPORARY as we only support a default output ...[defaultOutput].reduce( // eslint-disable-next-line @typescript-eslint/naming-convention - (outputs, { config: outputConfig, name, type, hosts, ca_sha256, api_key }) => { + (outputs, { config_yaml, name, type, hosts, ca_sha256, api_key }) => { + const configJs = config_yaml ? safeLoad(config_yaml) : {}; outputs[name] = { type, hosts, ca_sha256, api_key, - ...outputConfig, + ...configJs, }; if (options?.standalone) { @@ -526,7 +532,7 @@ class AgentPolicyService { // only add settings if not in standalone if (!standalone) { - let settings; + let settings: Settings; try { settings = await getSettings(soClient); } catch (error) { diff --git a/x-pack/plugins/ingest_manager/server/types/models/output.ts b/x-pack/plugins/ingest_manager/server/types/models/output.ts index 22a101ecd94b8e..c5aab6395adabe 100644 --- a/x-pack/plugins/ingest_manager/server/types/models/output.ts +++ b/x-pack/plugins/ingest_manager/server/types/models/output.ts @@ -17,6 +17,7 @@ const OutputBaseSchema = { fleet_enroll_username: schema.maybe(schema.string()), fleet_enroll_password: schema.maybe(schema.string()), config: schema.maybe(schema.recordOf(schema.string(), schema.any())), + config_yaml: schema.maybe(schema.string()), }; export const NewOutputSchema = schema.object({ diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/output.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/output.ts index 315923fd9f4017..bf8c98e6cabc98 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/output.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/output.ts @@ -20,5 +20,7 @@ export const PutOutputRequestSchema = { body: schema.object({ hosts: schema.maybe(schema.arrayOf(schema.uri({ scheme: ['http', 'https'] }))), ca_sha256: schema.maybe(schema.string()), + config: schema.maybe(schema.recordOf(schema.string(), schema.any())), + config_yaml: schema.maybe(schema.string()), }), }; diff --git a/x-pack/plugins/ingest_manager/server/types/rest_spec/settings.ts b/x-pack/plugins/ingest_manager/server/types/rest_spec/settings.ts index 35718491c92247..18cbd8437d8e25 100644 --- a/x-pack/plugins/ingest_manager/server/types/rest_spec/settings.ts +++ b/x-pack/plugins/ingest_manager/server/types/rest_spec/settings.ts @@ -23,5 +23,6 @@ export const PutSettingsRequestSchema = { ), kibana_ca_sha256: schema.maybe(schema.string()), has_seen_add_data_notice: schema.maybe(schema.boolean()), + additional_yaml_config: schema.maybe(schema.string()), }), }; From d5a19ec2e2c1a022d39557ec9bc5b0b3c3c33d6e Mon Sep 17 00:00:00 2001 From: Quynh Nguyen <43350163+qn895@users.noreply.github.com> Date: Tue, 6 Oct 2020 17:07:08 -0500 Subject: [PATCH 06/83] [ML] Fix jobs so it limit job menu actions for jobs that are closing (#79303) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../application/jobs/jobs_list/components/utils.js | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js index c1f6d75637ed41..bc85153928a4b6 100644 --- a/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js +++ b/x-pack/plugins/ml/public/application/jobs/jobs_list/components/utils.js @@ -38,7 +38,9 @@ export function loadFullJob(jobId) { } export function isStartable(jobs) { - return jobs.some((j) => j.datafeedState === DATAFEED_STATE.STOPPED); + return jobs.some( + (j) => j.datafeedState === DATAFEED_STATE.STOPPED && j.jobState !== JOB_STATE.CLOSING + ); } export function isStoppable(jobs) { @@ -49,7 +51,10 @@ export function isStoppable(jobs) { export function isClosable(jobs) { return jobs.some( - (j) => j.datafeedState === DATAFEED_STATE.STOPPED && j.jobState !== JOB_STATE.CLOSED + (j) => + j.datafeedState === DATAFEED_STATE.STOPPED && + j.jobState !== JOB_STATE.CLOSED && + j.jobState !== JOB_STATE.CLOSING ); } From 443049d23479c5fd275ed6a7fb956dc297520f22 Mon Sep 17 00:00:00 2001 From: Mikhail Shustov Date: Wed, 7 Oct 2020 01:18:51 +0300 Subject: [PATCH 07/83] use pretty logs when building type refs (#79711) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- src/dev/typescript/build_refs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/dev/typescript/build_refs.ts b/src/dev/typescript/build_refs.ts index 2cc8283111959e..fdc1dfbfffa0b1 100644 --- a/src/dev/typescript/build_refs.ts +++ b/src/dev/typescript/build_refs.ts @@ -29,7 +29,7 @@ export async function buildAllRefs(log: ToolingLog) { async function buildRefs(log: ToolingLog, projectPath: string) { try { log.debug(`Building TypeScript projects refs for ${projectPath}...`); - await execa(require.resolve('typescript/bin/tsc'), ['-b', projectPath]); + await execa(require.resolve('typescript/bin/tsc'), ['-b', projectPath, '--pretty']); } catch (e) { log.error(e); process.exit(1); From cb73df0bdcd1d1ffe6ed0f26d63f221d012e8adf Mon Sep 17 00:00:00 2001 From: Kevin Logan <56395104+kevinlog@users.noreply.github.com> Date: Tue, 6 Oct 2020 18:25:53 -0400 Subject: [PATCH 08/83] [SECURITY_SOLUTION] Task/add view agents link to enrolling (#79735) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- .../pages/endpoint_hosts/view/index.tsx | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx index 7639b878b9c5c9..ba3a90bbc7bdb1 100644 --- a/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx +++ b/x-pack/plugins/security_solution/public/management/pages/endpoint_hosts/view/index.tsx @@ -61,6 +61,7 @@ import { AdminSearchBar } from './components/search_bar'; import { AdministrationListPage } from '../../../components/administration_list_page'; import { useKibana } from '../../../../../../../../src/plugins/kibana_react/public'; import { APP_ID } from '../../../../../common/constants'; +import { LinkToApp } from '../../../../common/components/endpoint/link_to_app'; const EndpointListNavLink = memo<{ name: string; @@ -578,16 +579,32 @@ export const EndpointList = () => { <> {areEndpointsEnrolling && !hasErrorFindingTotals && ( <> - - } - /> + + + + + ), + }} + /> + )} From bd153c48cb8019e158ab67e24946b3073971202c Mon Sep 17 00:00:00 2001 From: Phillip Burch Date: Tue, 6 Oct 2020 17:27:30 -0500 Subject: [PATCH 09/83] Show callout for K8s jobs (#79610) --- .../components/ml/anomaly_detection/flyout_home.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx index 237639fbf7beb6..79a5cea12ef862 100644 --- a/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx +++ b/x-pack/plugins/infra/public/pages/metrics/inventory_view/components/ml/anomaly_detection/flyout_home.tsx @@ -109,7 +109,7 @@ export const FlyoutHome = (props: Props) => {
- {hostJobSummaries.length > 0 && ( + {(hostJobSummaries.length > 0 || k8sJobSummaries.length > 0) && ( <> 0} From 7031ea4f7b61c1dde96232cdb8ca52cac3fc9934 Mon Sep 17 00:00:00 2001 From: Ryland Herrick Date: Tue, 6 Oct 2020 18:13:31 -0500 Subject: [PATCH 10/83] [Security Solution][Detections] Rule Form: prevent creation of invalid scheduling parameters (#79577) * Clean up our component types * Clamp our Rule Schedule inputs to safe values * If the user enters a value > Number.MAX_SAFE_INTEGER, it will be updated to Number.MAX_SAFE_INTEGER * If the user enters non-numeric text, it will be updated to 0 * Ensure that we do not go below the default value 0 is not necessarily a reasonable default, but we already have a variable for the default value. * Update cypress interaction with schedule fields Now that we set defaults for bad values, it's no longer possible to "clear" the numeric input. However, to get roughly the same behavior we can instead select the current value and start typing. --- .../cypress/tasks/create_new_rule.ts | 6 +- .../rules/schedule_item_form/index.test.tsx | 84 ++++++++++++++++--- .../rules/schedule_item_form/index.tsx | 34 +++++--- 3 files changed, 98 insertions(+), 26 deletions(-) diff --git a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts index 079c18b6abe6ef..1433acd27c9307 100644 --- a/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts +++ b/x-pack/plugins/security_solution/cypress/tasks/create_new_rule.ts @@ -190,16 +190,16 @@ export const fillDefineCustomRuleWithImportedQueryAndContinue = ( ) => { cy.get(IMPORT_QUERY_FROM_SAVED_TIMELINE_LINK).click(); cy.get(TIMELINE(rule.timelineId)).click(); - cy.get(CUSTOM_QUERY_INPUT).invoke('text').should('eq', rule.customQuery); + cy.get(CUSTOM_QUERY_INPUT).should('have.text', rule.customQuery); cy.get(DEFINE_CONTINUE_BUTTON).should('exist').click({ force: true }); cy.get(CUSTOM_QUERY_INPUT).should('not.exist'); }; export const fillScheduleRuleAndContinue = (rule: CustomRule | MachineLearningRule) => { - cy.get(RUNS_EVERY_INTERVAL).clear().type(rule.runsEvery.interval); + cy.get(RUNS_EVERY_INTERVAL).type('{selectall}').type(rule.runsEvery.interval); cy.get(RUNS_EVERY_TIME_TYPE).select(rule.runsEvery.timeType); - cy.get(LOOK_BACK_INTERVAL).clear().type(rule.lookBack.interval); + cy.get(LOOK_BACK_INTERVAL).type('{selectAll}').type(rule.lookBack.interval); cy.get(LOOK_BACK_TIME_TYPE).select(rule.lookBack.timeType); }; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/schedule_item_form/index.test.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/schedule_item_form/index.test.tsx index 9dddd9e6c40851..7a5307f650ebc9 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/schedule_item_form/index.test.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/schedule_item_form/index.test.tsx @@ -5,27 +5,91 @@ */ import React from 'react'; -import { shallow } from 'enzyme'; +import { mount, shallow } from 'enzyme'; import { ScheduleItem } from './index'; -import { useFormFieldMock } from '../../../../common/mock'; +import { TestProviders, useFormFieldMock } from '../../../../common/mock'; describe('ScheduleItem', () => { it('renders correctly', () => { - const Component = () => { - const field = useFormFieldMock(); + const mockField = useFormFieldMock(); + const wrapper = shallow( + + ); - return ( + expect(wrapper.find('[data-test-subj="schedule-item"]')).toHaveLength(1); + }); + + it('accepts a large number via user input', () => { + const mockField = useFormFieldMock(); + const wrapper = mount( + + + + ); + + wrapper + .find('[data-test-subj="interval"]') + .last() + .simulate('change', { target: { value: '5000000' } }); + + expect(mockField.setValue).toHaveBeenCalledWith('5000000s'); + }); + + it('clamps a number value greater than MAX_SAFE_INTEGER to MAX_SAFE_INTEGER', () => { + const unsafeInput = '99999999999999999999999'; + + const mockField = useFormFieldMock(); + const wrapper = mount( + - ); - }; - const wrapper = shallow(); + + ); + + wrapper + .find('[data-test-subj="interval"]') + .last() + .simulate('change', { target: { value: unsafeInput } }); + + const expectedValue = `${Number.MAX_SAFE_INTEGER}s`; + expect(mockField.setValue).toHaveBeenCalledWith(expectedValue); + }); + + it('converts a non-numeric value to 0', () => { + const unsafeInput = 'this is not a number'; + + const mockField = useFormFieldMock(); + const wrapper = mount( + + + + ); + + wrapper + .find('[data-test-subj="interval"]') + .last() + .simulate('change', { target: { value: unsafeInput } }); - expect(wrapper.dive().find('[data-test-subj="schedule-item"]')).toHaveLength(1); + expect(mockField.setValue).toHaveBeenCalledWith('0s'); }); }); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/schedule_item_form/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/schedule_item_form/index.tsx index ae012774fa30d3..10d6325b99cbbc 100644 --- a/x-pack/plugins/security_solution/public/detections/components/rules/schedule_item_form/index.tsx +++ b/x-pack/plugins/security_solution/public/detections/components/rules/schedule_item_form/index.tsx @@ -21,7 +21,7 @@ import { FieldHook, getFieldValidityAndErrorMessage } from '../../../../shared_i import * as I18n from './translations'; interface ScheduleItemProps { - field: FieldHook; + field: FieldHook; dataTestSubj: string; idAria: string; isDisabled: boolean; @@ -62,6 +62,15 @@ const MyEuiSelect = styled(EuiSelect)` width: auto; `; +const getNumberFromUserInput = (input: string, defaultValue = 0): number => { + const number = parseInt(input, 10); + if (Number.isNaN(number)) { + return defaultValue; + } else { + return Math.min(number, Number.MAX_SAFE_INTEGER); + } +}; + export const ScheduleItem = ({ dataTestSubj, field, @@ -72,30 +81,29 @@ export const ScheduleItem = ({ const [timeType, setTimeType] = useState('s'); const [timeVal, setTimeVal] = useState(0); const { isInvalid, errorMessage } = getFieldValidityAndErrorMessage(field); + const { value, setValue } = field; const onChangeTimeType = useCallback( (e) => { setTimeType(e.target.value); - field.setValue(`${timeVal}${e.target.value}`); + setValue(`${timeVal}${e.target.value}`); }, - // eslint-disable-next-line react-hooks/exhaustive-deps - [timeVal] + [setValue, timeVal] ); const onChangeTimeVal = useCallback( (e) => { - const sanitizedValue: number = parseInt(e.target.value, 10); + const sanitizedValue = getNumberFromUserInput(e.target.value, minimumValue); setTimeVal(sanitizedValue); - field.setValue(`${sanitizedValue}${timeType}`); + setValue(`${sanitizedValue}${timeType}`); }, - // eslint-disable-next-line react-hooks/exhaustive-deps - [timeType] + [minimumValue, setValue, timeType] ); useEffect(() => { - if (field.value !== `${timeVal}${timeType}`) { - const filterTimeVal = (field.value as string).match(/\d+/g); - const filterTimeType = (field.value as string).match(/[a-zA-Z]+/g); + if (value !== `${timeVal}${timeType}`) { + const filterTimeVal = value.match(/\d+/g); + const filterTimeType = value.match(/[a-zA-Z]+/g); if ( !isEmpty(filterTimeVal) && filterTimeVal != null && @@ -113,8 +121,7 @@ export const ScheduleItem = ({ setTimeType(filterTimeType[0]); } } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [field.value]); + }, [timeType, timeVal, value]); // EUI missing some props const rest = { disabled: isDisabled }; @@ -157,6 +164,7 @@ export const ScheduleItem = ({ Date: Tue, 6 Oct 2020 19:40:28 -0400 Subject: [PATCH 11/83] send protocol separately to agent as part of full agent policy (#79781) --- .../ingest_manager/common/types/models/agent_policy.ts | 1 + .../server/services/agent_policy.test.ts | 9 ++++++--- .../ingest_manager/server/services/agent_policy.ts | 10 ++++++++-- .../apps/endpoint/policy_details.ts | 2 +- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/x-pack/plugins/ingest_manager/common/types/models/agent_policy.ts b/x-pack/plugins/ingest_manager/common/types/models/agent_policy.ts index b3b3004f4fc5d6..8d8344aed6c4cb 100644 --- a/x-pack/plugins/ingest_manager/common/types/models/agent_policy.ts +++ b/x-pack/plugins/ingest_manager/common/types/models/agent_policy.ts @@ -64,6 +64,7 @@ export interface FullAgentPolicy { fleet?: { kibana: { hosts: string[]; + protocol: string; }; }; inputs: FullAgentPolicyInput[]; diff --git a/x-pack/plugins/ingest_manager/server/services/agent_policy.test.ts b/x-pack/plugins/ingest_manager/server/services/agent_policy.test.ts index d9dffa03b2290f..d247b35c089e52 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_policy.test.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_policy.test.ts @@ -81,7 +81,8 @@ describe('agent policy', () => { revision: 1, fleet: { kibana: { - hosts: ['http://localhost:5603'], + hosts: ['localhost:5603'], + protocol: 'http', }, }, agent: { @@ -115,7 +116,8 @@ describe('agent policy', () => { revision: 1, fleet: { kibana: { - hosts: ['http://localhost:5603'], + hosts: ['localhost:5603'], + protocol: 'http', }, }, agent: { @@ -150,7 +152,8 @@ describe('agent policy', () => { revision: 1, fleet: { kibana: { - hosts: ['http://localhost:5603'], + hosts: ['localhost:5603'], + protocol: 'http', }, }, agent: { diff --git a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts index 37d5d49fcbfeff..f1dcc7e5d6c99a 100644 --- a/x-pack/plugins/ingest_manager/server/services/agent_policy.ts +++ b/x-pack/plugins/ingest_manager/server/services/agent_policy.ts @@ -538,10 +538,16 @@ class AgentPolicyService { } catch (error) { throw new Error('Default settings is not setup'); } - if (!settings.kibana_urls) throw new Error('kibana_urls is missing'); + if (!settings.kibana_urls || !settings.kibana_urls.length) + throw new Error('kibana_urls is missing'); + const hostsWithoutProtocol = settings.kibana_urls.map((url) => { + const parsedURL = new URL(url); + return `${parsedURL.host}${parsedURL.pathname !== '/' ? parsedURL.pathname : ''}`; + }); fullAgentPolicy.fleet = { kibana: { - hosts: settings.kibana_urls, + protocol: new URL(settings.kibana_urls[0]).protocol.replace(':', ''), + hosts: hostsWithoutProtocol, }, }; } diff --git a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts index b15fab96470e0e..927255a93624ea 100644 --- a/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts +++ b/x-pack/test/security_solution_endpoint/apps/endpoint/policy_details.ts @@ -25,7 +25,6 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { const { protocol, hostname, port } = kbnTestServer; const kibanaUrl = Url.format({ - protocol, hostname, port, }); @@ -237,6 +236,7 @@ export default function ({ getPageObjects, getService }: FtrProviderContext) { fleet: { kibana: { hosts: [kibanaUrl], + protocol, }, }, revision: 3, From e4fc48cd5f94f9d2d3bbb16a51a0c6a1e573d1a1 Mon Sep 17 00:00:00 2001 From: Yara Tercero Date: Tue, 6 Oct 2020 19:48:04 -0400 Subject: [PATCH 12/83] [Security Solution][Detections] - Rule creation query preview (#78985) ### Summary This PR introduces a preview histogram feature inside the rule creation workflow, that gives users insight to how effective the rule is for triggering alerts the user would like to see. --- .../common/detection_engine/types.ts | 45 ++ .../matrix_histogram/index.ts | 1 + .../common/components/charts/barchart.tsx | 5 + .../common/components/charts/common.tsx | 3 + .../components/matrix_histogram/index.tsx | 21 +- .../components/matrix_histogram/types.ts | 2 + .../components/matrix_histogram/utils.ts | 3 + .../containers/matrix_histogram/index.ts | 17 +- .../public/common/hooks/eql/api.ts | 71 ++ .../public/common/hooks/eql/helpers.test.ts | 683 ++++++++++++++++++ .../public/common/hooks/eql/helpers.ts | 190 +++++ .../public/common/hooks/eql/index.ts | 7 + .../public/common/hooks/eql/types.ts | 20 + .../common/hooks/eql/use_eql_preview.ts | 12 + .../rules/query_preview/eql_histogram.tsx | 78 ++ .../components/rules/query_preview/helpers.ts | 109 +++ .../components/rules/query_preview/index.tsx | 258 +++++++ .../rules/query_preview/non_eql_histogram.tsx | 80 ++ .../query_preview/threshold_histogram.tsx | 102 +++ .../rules/query_preview/translations.ts | 126 ++++ .../rules/step_define_rule/index.tsx | 50 +- .../routes/__mocks__/request_responses.ts | 3 +- .../signals/build_bulk_body.ts | 3 +- .../signals/filter_events_with_list.ts | 3 +- .../lib/detection_engine/signals/types.ts | 20 +- .../server/lib/hosts/types.ts | 3 +- .../security_solution/server/lib/types.ts | 76 +- .../events/__mocks__/index.ts | 109 +++ .../events/query.events_histogram.dsl.test.ts | 19 +- .../events/query.events_histogram.dsl.ts | 20 + 30 files changed, 2055 insertions(+), 84 deletions(-) create mode 100644 x-pack/plugins/security_solution/public/common/hooks/eql/helpers.test.ts create mode 100644 x-pack/plugins/security_solution/public/common/hooks/eql/helpers.ts create mode 100644 x-pack/plugins/security_solution/public/common/hooks/eql/index.ts create mode 100644 x-pack/plugins/security_solution/public/common/hooks/eql/types.ts create mode 100644 x-pack/plugins/security_solution/public/common/hooks/eql/use_eql_preview.ts create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/query_preview/eql_histogram.tsx create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/query_preview/helpers.ts create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/query_preview/non_eql_histogram.tsx create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/query_preview/threshold_histogram.tsx create mode 100644 x-pack/plugins/security_solution/public/detections/components/rules/query_preview/translations.ts diff --git a/x-pack/plugins/security_solution/common/detection_engine/types.ts b/x-pack/plugins/security_solution/common/detection_engine/types.ts index e7aab2fa5d219e..bf57d7cb7815af 100644 --- a/x-pack/plugins/security_solution/common/detection_engine/types.ts +++ b/x-pack/plugins/security_solution/common/detection_engine/types.ts @@ -8,3 +8,48 @@ import { AlertAction } from '../../../alerts/common'; export type RuleAlertAction = Omit & { action_type_id: string; }; + +export type SearchTypes = + | string + | string[] + | number + | number[] + | boolean + | boolean[] + | object + | object[] + | undefined; + +export interface Explanation { + value: number; + description: string; + details: Explanation[]; +} + +export interface TotalValue { + value: number; + relation: string; +} + +export interface BaseHit { + _index: string; + _id: string; + _source: T; +} + +export interface EqlSequence { + join_keys: SearchTypes[]; + events: Array>; +} + +export interface EqlSearchResponse { + is_partial: boolean; + is_running: boolean; + took: number; + timed_out: boolean; + hits: { + total: TotalValue; + sequences?: Array>; + events?: Array>; + }; +} diff --git a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts index 238300801cfc6a..0217c48668fb90 100644 --- a/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/common/search_strategy/security_solution/matrix_histogram/index.ts @@ -35,6 +35,7 @@ export interface MatrixHistogramRequestOptions extends RequestBasicOptions { timerange: TimerangeInput; histogramType: MatrixHistogramType; stackByField: string; + threshold?: { field: string | undefined; value: number } | undefined; inspect?: Maybe; } diff --git a/x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx b/x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx index 80fc1b45970812..fb1ed956dfc52c 100644 --- a/x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx +++ b/x-pack/plugins/security_solution/public/common/components/charts/barchart.tsx @@ -51,11 +51,13 @@ const checkIfAnyValidSeriesExist = ( export const BarChartBaseComponent = ({ data, forceHiddenLegend = false, + yAxisTitle, ...chartConfigs }: { data: ChartSeriesData[]; width: string | null | undefined; height: string | null | undefined; + yAxisTitle?: string | undefined; configs?: ChartSeriesConfigs | undefined; forceHiddenLegend?: boolean; }) => { @@ -115,6 +117,7 @@ export const BarChartBaseComponent = ({ }, }} tickFormat={yTickFormatter} + title={yAxisTitle} /> ) : null; @@ -158,6 +161,7 @@ export const BarChartComponent: React.FC = ({ [barChart, stackByField, timelineId] ); + const yAxisTitle = get('yAxisTitle', configs); const customHeight = get('customHeight', configs); const customWidth = get('customWidth', configs); const chartHeight = getChartHeight(customHeight, height); @@ -170,6 +174,7 @@ export const BarChartComponent: React.FC = ({ ; } diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx index e7d7e60a3c4088..7395100784d51b 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/index.tsx @@ -34,6 +34,7 @@ export type MatrixHistogramComponentProps = MatrixHistogramProps & defaultStackByOption: MatrixHistogramOption; errorMessage: string; headerChildren?: React.ReactNode; + footerChildren?: React.ReactNode; hideHistogramIfEmpty?: boolean; histogramType: MatrixHistogramType; id: string; @@ -47,6 +48,7 @@ export type MatrixHistogramComponentProps = MatrixHistogramProps & subtitle?: string | GetSubTitle; timelineId?: string; title: string | GetTitle; + yTitle?: string | undefined; }; const DEFAULT_PANEL_HEIGHT = 300; @@ -68,6 +70,7 @@ export const MatrixHistogramComponent: React.FC = errorMessage, filterQuery, headerChildren, + footerChildren, histogramType, hideHistogramIfEmpty = false, id, @@ -86,6 +89,7 @@ export const MatrixHistogramComponent: React.FC = title, titleSize, yTickFormatter, + yTitle, }) => { const dispatch = useDispatch(); const handleBrushEnd = useCallback( @@ -114,8 +118,18 @@ export const MatrixHistogramComponent: React.FC = onBrushEnd: handleBrushEnd, yTickFormatter, showLegend, + yTitle, }), - [chartHeight, startDate, legendPosition, endDate, handleBrushEnd, yTickFormatter, showLegend] + [ + chartHeight, + startDate, + legendPosition, + endDate, + handleBrushEnd, + yTickFormatter, + showLegend, + yTitle, + ] ); const [isInitialLoading, setIsInitialLoading] = useState(true); const [selectedStackByOption, setSelectedStackByOption] = useState( @@ -229,6 +243,11 @@ export const MatrixHistogramComponent: React.FC = timelineId={timelineId} /> )} + {footerChildren != null && ( + + {footerChildren} + + )} {showSpacer && } diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts index 9a892110bde43d..4c04a4cca9f827 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/types.ts @@ -70,6 +70,7 @@ export interface MatrixHistogramQueryProps { stackByField: string; startDate: string; histogramType: MatrixHistogramType; + threshold?: { field: string | undefined; value: number } | undefined; } export interface MatrixHistogramProps extends MatrixHistogramBasicProps { @@ -104,6 +105,7 @@ export interface BarchartConfigs { yTickFormatter: TickFormatter; tickSize: number; }; + yAxisTitle: string | undefined; settings: { legendPosition: Position; onBrushEnd: UpdateDateRange; diff --git a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/utils.ts b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/utils.ts index 5b5b56cf0ec45d..d1af29d7da27c7 100644 --- a/x-pack/plugins/security_solution/public/common/components/matrix_histogram/utils.ts +++ b/x-pack/plugins/security_solution/public/common/components/matrix_histogram/utils.ts @@ -19,6 +19,7 @@ interface GetBarchartConfigsProps { onBrushEnd: UpdateDateRange; yTickFormatter?: (value: number) => string; showLegend?: boolean; + yTitle?: string | undefined; } export const DEFAULT_CHART_HEIGHT = 174; @@ -32,6 +33,7 @@ export const getBarchartConfigs = ({ onBrushEnd, yTickFormatter, showLegend, + yTitle, }: GetBarchartConfigsProps): BarchartConfigs => ({ series: { xScaleType: ScaleType.Time, @@ -43,6 +45,7 @@ export const getBarchartConfigs = ({ yTickFormatter: yTickFormatter != null ? yTickFormatter : DEFAULT_Y_TICK_FORMATTER, tickSize: 8, }, + yAxisTitle: yTitle, settings: { legendPosition: legendPosition ?? Position.Right, onBrushEnd, diff --git a/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts b/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts index 54e349fe3e9263..7c6a110f56b81c 100644 --- a/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts +++ b/x-pack/plugins/security_solution/public/common/containers/matrix_histogram/index.ts @@ -5,7 +5,7 @@ */ import deepEqual from 'fast-deep-equal'; -import { noop } from 'lodash/fp'; +import { getOr, noop } from 'lodash/fp'; import { useCallback, useEffect, useRef, useState } from 'react'; import { MatrixHistogramQueryProps } from '../../components/matrix_histogram/types'; @@ -32,6 +32,10 @@ export interface UseMatrixHistogramArgs { inspect: InspectResponse; refetch: inputsModel.Refetch; totalCount: number; + buckets: Array<{ + key: string; + doc_count: number; + }>; } const ID = 'matrixHistogramQuery'; @@ -44,6 +48,7 @@ export const useMatrixHistogram = ({ indexNames, stackByField, startDate, + threshold, }: MatrixHistogramQueryProps): [boolean, UseMatrixHistogramArgs] => { const { data, notifications } = useKibana().services; const refetch = useRef(noop); @@ -63,6 +68,7 @@ export const useMatrixHistogram = ({ to: endDate, }, stackByField, + threshold, }); const [matrixHistogramResponse, setMatrixHistogramResponse] = useState({ @@ -73,6 +79,7 @@ export const useMatrixHistogram = ({ }, refetch: refetch.current, totalCount: -1, + buckets: [], }); const hostsSearch = useCallback( @@ -91,6 +98,10 @@ export const useMatrixHistogram = ({ next: (response) => { if (isCompleteResponse(response)) { if (!didCancel) { + const histogramBuckets: Array<{ + key: string; + doc_count: number; + }> = getOr([], 'rawResponse.aggregations.eventActionGroup.buckets', response); setLoading(false); setMatrixHistogramResponse((prevResponse) => ({ ...prevResponse, @@ -98,6 +109,7 @@ export const useMatrixHistogram = ({ inspect: getInspectResponse(response, prevResponse.inspect), refetch: refetch.current, totalCount: response.totalCount, + buckets: histogramBuckets, })); } searchSubscription$.unsubscribe(); @@ -144,13 +156,14 @@ export const useMatrixHistogram = ({ to: endDate, }, stackByField, + threshold, }; if (!deepEqual(prevRequest, myRequest)) { return myRequest; } return prevRequest; }); - }, [indexNames, endDate, filterQuery, startDate, stackByField, histogramType]); + }, [indexNames, endDate, filterQuery, startDate, stackByField, histogramType, threshold]); useEffect(() => { hostsSearch(matrixHistogramRequest); diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts index 11fe79910bc876..5e69674e368920 100644 --- a/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts +++ b/x-pack/plugins/security_solution/public/common/hooks/eql/api.ts @@ -3,11 +3,21 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ +import { Unit } from '@elastic/datemath'; import { HttpStart } from '../../../../../../../src/core/public'; import { DETECTION_ENGINE_EQL_VALIDATION_URL } from '../../../../common/constants'; import { EqlValidationSchema as EqlValidationRequest } from '../../../../common/detection_engine/schemas/request/eql_validation_schema'; import { EqlValidationSchema as EqlValidationResponse } from '../../../../common/detection_engine/schemas/response/eql_validation_schema'; +import { DataPublicPluginStart } from '../../../../../../../src/plugins/data/public'; +import { + EqlSearchStrategyRequest, + EqlSearchStrategyResponse, +} from '../../../../../data_enhanced/common'; +import { getEqlAggsData, getSequenceAggs } from './helpers'; +import { EqlPreviewResponse, Source } from './types'; +import { hasEqlSequenceQuery } from '../../../../common/detection_engine/utils'; +import { EqlSearchResponse } from '../../../../common/detection_engine/types'; interface ApiParams { http: HttpStart; @@ -29,3 +39,64 @@ export const validateEql = async ({ signal, }); }; + +interface AggsParams extends EqlValidationRequest { + data: DataPublicPluginStart; + interval: Unit; + fromTime: string; + toTime: string; + signal: AbortSignal; +} + +export const getEqlPreview = async ({ + data, + index, + interval, + query, + fromTime, + toTime, + signal, +}: AggsParams): Promise => { + try { + const response = await data.search + .search>>( + { + params: { + // @ts-expect-error allow_no_indices is missing on EqlSearch + allow_no_indices: true, + index: index.join(), + body: { + filter: { + range: { + '@timestamp': { + gte: toTime, + lte: fromTime, + format: 'strict_date_optional_time', + }, + }, + }, + query, + // EQL requires a cap, otherwise it defaults to 10 + // It also sorts on ascending order, capping it at + // something smaller like 20, made it so that some of + // the more recent events weren't returned + size: 100, + }, + }, + }, + { + strategy: 'eql', + abortSignal: signal, + } + ) + .toPromise(); + + if (hasEqlSequenceQuery(query)) { + return getSequenceAggs(response, interval, toTime, fromTime); + } else { + return getEqlAggsData(response, interval, toTime, fromTime); + } + } catch (err) { + throw new Error(JSON.stringify(err)); + } +}; diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/helpers.test.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/helpers.test.ts new file mode 100644 index 00000000000000..1418c1155877b0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/eql/helpers.test.ts @@ -0,0 +1,683 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import dateMath from '@elastic/datemath'; + +import { EqlSearchStrategyResponse } from '../../../../../data_enhanced/common'; +import { Source } from './types'; +import { EqlSearchResponse } from '../../../../common/detection_engine/types'; + +import { + calculateBucketForHour, + calculateBucketForDay, + getEqlAggsData, + createIntervalArray, + getInterval, + getSequenceAggs, +} from './helpers'; + +const getMockResponse = (): EqlSearchStrategyResponse> => + ({ + id: 'some-id', + rawResponse: { + body: { + hits: { + events: [ + { + _index: 'index', + _id: '1', + _source: { + '@timestamp': '2020-10-04T15:16:54.368707900Z', + }, + }, + { + _index: 'index', + _id: '2', + _source: { + '@timestamp': '2020-10-04T15:50:54.368707900Z', + }, + }, + { + _index: 'index', + _id: '3', + _source: { + '@timestamp': '2020-10-04T15:06:54.368707900Z', + }, + }, + { + _index: 'index', + _id: '4', + _source: { + '@timestamp': '2020-10-04T15:15:54.368707900Z', + }, + }, + ], + total: { + value: 4, + relation: '', + }, + }, + }, + meta: { + request: { + params: { + method: 'GET', + path: '/_eql/search/', + }, + options: {}, + id: '', + }, + }, + statusCode: 200, + }, + } as EqlSearchStrategyResponse>); + +const getMockSequenceResponse = (): EqlSearchStrategyResponse> => + (({ + id: 'some-id', + rawResponse: { + body: { + hits: { + sequences: [ + { + join_keys: [], + events: [ + { + _index: 'index', + _id: '1', + _source: { + '@timestamp': '2020-10-04T15:16:54.368707900Z', + }, + }, + { + _index: 'index', + _id: '2', + _source: { + '@timestamp': '2020-10-04T15:50:54.368707900Z', + }, + }, + ], + }, + { + join_keys: [], + events: [ + { + _index: 'index', + _id: '3', + _source: { + '@timestamp': '2020-10-04T15:06:54.368707900Z', + }, + }, + { + _index: 'index', + _id: '4', + _source: { + '@timestamp': '2020-10-04T15:15:54.368707900Z', + }, + }, + ], + }, + ], + total: { + value: 4, + relation: '', + }, + }, + }, + meta: { + request: { + params: { + method: 'GET', + path: '/_eql/search/', + }, + options: {}, + id: '', + }, + }, + statusCode: 200, + }, + } as unknown) as EqlSearchStrategyResponse>); + +describe('eql/helpers', () => { + describe('calculateBucketForHour', () => { + test('returns 2 if event occured within 2 minutes of "now"', () => { + const diff = calculateBucketForHour( + Number(dateMath.parse('now-1m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(2); + }); + + test('returns 10 if event occured within 8-10 minutes of "now"', () => { + const diff = calculateBucketForHour( + Number(dateMath.parse('now-9m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(10); + }); + + test('returns 16 if event occured within 10-15 minutes of "now"', () => { + const diff = calculateBucketForHour( + Number(dateMath.parse('now-15m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(16); + }); + + test('returns 60 if event occured within 58-60 minutes of "now"', () => { + const diff = calculateBucketForHour( + Number(dateMath.parse('now-59m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(60); + }); + + test('returns exact time difference if it is a multiple of 2', () => { + const diff = calculateBucketForHour( + Number(dateMath.parse('now-20m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(20); + }); + + test('returns 0 if times are equal', () => { + const diff = calculateBucketForHour( + Number(dateMath.parse('now')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(0); + }); + }); + + describe('calculateBucketForDay', () => { + test('returns 0 if two dates are equivalent', () => { + const diff = calculateBucketForDay( + Number(dateMath.parse('now')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(0); + }); + + test('returns 1 if event occured within 60 minutes of "now"', () => { + const diff = calculateBucketForDay( + Number(dateMath.parse('now-40m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(1); + }); + + test('returns 2 if event occured 60-120 minutes from "now"', () => { + const diff = calculateBucketForDay( + Number(dateMath.parse('now-120m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(2); + }); + + test('returns 3 if event occured 120-180 minutes from "now', () => { + const diff = calculateBucketForDay( + Number(dateMath.parse('now-121m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(3); + }); + + test('returns 4 if event occured 180-240 minutes from "now', () => { + const diff = calculateBucketForDay( + Number(dateMath.parse('now-220m')?.format('x')), + Number(dateMath.parse('now')?.format('x')) + ); + + expect(diff).toEqual(4); + }); + }); + + describe('getEqlAggsData', () => { + test('it returns results bucketed into 5 min intervals when range is "h"', () => { + const mockResponse = getMockResponse(); + + const aggs = getEqlAggsData( + mockResponse, + 'h', + '2020-10-04T15:00:00.368707900Z', + '2020-10-04T16:00:00.368707900Z' + ); + + expect(aggs.data).toHaveLength(31); + expect(aggs.data).toEqual([ + { g: 'hits', x: 1601827200368, y: 0 }, + { g: 'hits', x: 1601827080368, y: 0 }, + { g: 'hits', x: 1601826960368, y: 0 }, + { g: 'hits', x: 1601826840368, y: 0 }, + { g: 'hits', x: 1601826720368, y: 0 }, + { g: 'hits', x: 1601826600368, y: 1 }, + { g: 'hits', x: 1601826480368, y: 0 }, + { g: 'hits', x: 1601826360368, y: 0 }, + { g: 'hits', x: 1601826240368, y: 0 }, + { g: 'hits', x: 1601826120368, y: 0 }, + { g: 'hits', x: 1601826000368, y: 0 }, + { g: 'hits', x: 1601825880368, y: 0 }, + { g: 'hits', x: 1601825760368, y: 0 }, + { g: 'hits', x: 1601825640368, y: 0 }, + { g: 'hits', x: 1601825520368, y: 0 }, + { g: 'hits', x: 1601825400368, y: 0 }, + { g: 'hits', x: 1601825280368, y: 0 }, + { g: 'hits', x: 1601825160368, y: 0 }, + { g: 'hits', x: 1601825040368, y: 0 }, + { g: 'hits', x: 1601824920368, y: 0 }, + { g: 'hits', x: 1601824800368, y: 0 }, + { g: 'hits', x: 1601824680368, y: 0 }, + { g: 'hits', x: 1601824560368, y: 2 }, + { g: 'hits', x: 1601824440368, y: 0 }, + { g: 'hits', x: 1601824320368, y: 0 }, + { g: 'hits', x: 1601824200368, y: 0 }, + { g: 'hits', x: 1601824080368, y: 0 }, + { g: 'hits', x: 1601823960368, y: 1 }, + { g: 'hits', x: 1601823840368, y: 0 }, + { g: 'hits', x: 1601823720368, y: 0 }, + { g: 'hits', x: 1601823600368, y: 0 }, + ]); + }); + + test('it returns results bucketed into 1 hour intervals when range is "d"', () => { + const mockResponse = getMockResponse(); + const response = { + ...mockResponse, + rawResponse: { + ...mockResponse.rawResponse, + body: { + is_partial: false, + is_running: false, + timed_out: false, + took: 15, + hits: { + events: [ + { + _index: 'index', + _id: '1', + _source: { + '@timestamp': '2020-10-04T15:16:54.368707900Z', + }, + }, + { + _index: 'index', + _id: '2', + _source: { + '@timestamp': '2020-10-04T05:50:54.368707900Z', + }, + }, + { + _index: 'index', + _id: '3', + _source: { + '@timestamp': '2020-10-04T18:06:54.368707900Z', + }, + }, + { + _index: 'index', + _id: '4', + _source: { + '@timestamp': '2020-10-04T23:15:54.368707900Z', + }, + }, + ], + total: { + value: 4, + relation: '', + }, + }, + }, + }, + }; + + const aggs = getEqlAggsData( + response, + 'd', + '2020-10-03T23:50:00.368707900Z', + '2020-10-04T23:50:00.368707900Z' + ); + + expect(aggs.data).toHaveLength(25); + expect(aggs.data).toEqual([ + { g: 'hits', x: 1601855400368, y: 0 }, + { g: 'hits', x: 1601851800368, y: 1 }, + { g: 'hits', x: 1601848200368, y: 0 }, + { g: 'hits', x: 1601844600368, y: 0 }, + { g: 'hits', x: 1601841000368, y: 0 }, + { g: 'hits', x: 1601837400368, y: 0 }, + { g: 'hits', x: 1601833800368, y: 1 }, + { g: 'hits', x: 1601830200368, y: 0 }, + { g: 'hits', x: 1601826600368, y: 0 }, + { g: 'hits', x: 1601823000368, y: 1 }, + { g: 'hits', x: 1601819400368, y: 0 }, + { g: 'hits', x: 1601815800368, y: 0 }, + { g: 'hits', x: 1601812200368, y: 0 }, + { g: 'hits', x: 1601808600368, y: 0 }, + { g: 'hits', x: 1601805000368, y: 0 }, + { g: 'hits', x: 1601801400368, y: 0 }, + { g: 'hits', x: 1601797800368, y: 0 }, + { g: 'hits', x: 1601794200368, y: 0 }, + { g: 'hits', x: 1601790600368, y: 1 }, + { g: 'hits', x: 1601787000368, y: 0 }, + { g: 'hits', x: 1601783400368, y: 0 }, + { g: 'hits', x: 1601779800368, y: 0 }, + { g: 'hits', x: 1601776200368, y: 0 }, + { g: 'hits', x: 1601772600368, y: 0 }, + { g: 'hits', x: 1601769000368, y: 0 }, + ]); + }); + + test('it correctly returns total hits', () => { + const mockResponse = getMockResponse(); + + const aggs = getEqlAggsData( + mockResponse, + 'h', + '2020-10-04T15:00:00.368707900Z', + '2020-10-04T16:00:00.368707900Z' + ); + + expect(aggs.totalCount).toEqual(4); + }); + + test('it returns array with each item having a "total" of 0 if response returns no hits', () => { + const mockResponse = getMockResponse(); + const response = { + ...mockResponse, + rawResponse: { + ...mockResponse.rawResponse, + body: { + id: 'some-id', + is_partial: false, + is_running: false, + timed_out: false, + took: 15, + hits: { + total: { + value: 0, + relation: '', + }, + }, + }, + }, + }; + + const aggs = getEqlAggsData( + response, + 'h', + '2020-10-04T15:00:00.368707900Z', + '2020-10-04T16:00:00.368707900Z' + ); + + expect(aggs).toEqual({ + data: [ + { g: 'hits', x: 1601827200368, y: 0 }, + { g: 'hits', x: 1601827080368, y: 0 }, + { g: 'hits', x: 1601826960368, y: 0 }, + { g: 'hits', x: 1601826840368, y: 0 }, + { g: 'hits', x: 1601826720368, y: 0 }, + { g: 'hits', x: 1601826600368, y: 0 }, + { g: 'hits', x: 1601826480368, y: 0 }, + { g: 'hits', x: 1601826360368, y: 0 }, + { g: 'hits', x: 1601826240368, y: 0 }, + { g: 'hits', x: 1601826120368, y: 0 }, + { g: 'hits', x: 1601826000368, y: 0 }, + { g: 'hits', x: 1601825880368, y: 0 }, + { g: 'hits', x: 1601825760368, y: 0 }, + { g: 'hits', x: 1601825640368, y: 0 }, + { g: 'hits', x: 1601825520368, y: 0 }, + { g: 'hits', x: 1601825400368, y: 0 }, + { g: 'hits', x: 1601825280368, y: 0 }, + { g: 'hits', x: 1601825160368, y: 0 }, + { g: 'hits', x: 1601825040368, y: 0 }, + { g: 'hits', x: 1601824920368, y: 0 }, + { g: 'hits', x: 1601824800368, y: 0 }, + { g: 'hits', x: 1601824680368, y: 0 }, + { g: 'hits', x: 1601824560368, y: 0 }, + { g: 'hits', x: 1601824440368, y: 0 }, + { g: 'hits', x: 1601824320368, y: 0 }, + { g: 'hits', x: 1601824200368, y: 0 }, + { g: 'hits', x: 1601824080368, y: 0 }, + { g: 'hits', x: 1601823960368, y: 0 }, + { g: 'hits', x: 1601823840368, y: 0 }, + { g: 'hits', x: 1601823720368, y: 0 }, + { g: 'hits', x: 1601823600368, y: 0 }, + ], + gte: '2020-10-04T15:00:00.368707900Z', + inspect: { + dsl: [JSON.stringify(response.rawResponse.meta.request.params, null, 2)], + response: [JSON.stringify(response.rawResponse.body, null, 2)], + }, + lte: '2020-10-04T16:00:00.368707900Z', + totalCount: 0, + warnings: [], + }); + }); + }); + + describe('createIntervalArray', () => { + test('returns array of 12 numbers from 0 to 60 by 5', () => { + const arrayOfNumbers = createIntervalArray(0, 12, 5); + expect(arrayOfNumbers).toEqual([0, 5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60]); + }); + + test('returns array of 30 numbers from 0 to 60 by 2', () => { + const arrayOfNumbers = createIntervalArray(0, 30, 2); + expect(arrayOfNumbers).toEqual([ + 0, + 2, + 4, + 6, + 8, + 10, + 12, + 14, + 16, + 18, + 20, + 22, + 24, + 26, + 28, + 30, + 32, + 34, + 36, + 38, + 40, + 42, + 44, + 46, + 48, + 50, + 52, + 54, + 56, + 58, + 60, + ]); + }); + + test('returns array of 30 numbers from start param to end param if multiplier is 1', () => { + const arrayOfNumbers = createIntervalArray(0, 12, 1); + expect(arrayOfNumbers).toEqual([0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]); + }); + }); + + describe('getInterval', () => { + test('returns object with 2 minute interval keys if range is "h"', () => { + const intervals = getInterval('h', 1601856270140); + const keys = Object.keys(intervals); + expect(keys).toEqual([ + '0', + '2', + '4', + '6', + '8', + '10', + '12', + '14', + '16', + '18', + '20', + '22', + '24', + '26', + '28', + '30', + '32', + '34', + '36', + '38', + '40', + '42', + '44', + '46', + '48', + '50', + '52', + '54', + '56', + '58', + '60', + ]); + }); + + test('returns object with 2 minute interval timestamps if range is "h"', () => { + const intervals = getInterval('h', 1601856270140); + const timestamps = Object.keys(intervals).map((key) => intervals[key].timestamp); + expect(timestamps).toEqual([ + '1601856270140', + '1601856150140', + '1601856030140', + '1601855910140', + '1601855790140', + '1601855670140', + '1601855550140', + '1601855430140', + '1601855310140', + '1601855190140', + '1601855070140', + '1601854950140', + '1601854830140', + '1601854710140', + '1601854590140', + '1601854470140', + '1601854350140', + '1601854230140', + '1601854110140', + '1601853990140', + '1601853870140', + '1601853750140', + '1601853630140', + '1601853510140', + '1601853390140', + '1601853270140', + '1601853150140', + '1601853030140', + '1601852910140', + '1601852790140', + '1601852670140', + ]); + }); + + test('returns object with 1 hour interval keys if range is "d"', () => { + const intervals = getInterval('d', 1601856270140); + const keys = Object.keys(intervals); + expect(keys).toEqual([ + '0', + '1', + '2', + '3', + '4', + '5', + '6', + '7', + '8', + '9', + '10', + '11', + '12', + '13', + '14', + '15', + '16', + '17', + '18', + '19', + '20', + '21', + '22', + '23', + '24', + ]); + }); + + test('returns object with 1 hour interval timestamps if range is "d"', () => { + const intervals = getInterval('d', 1601856270140); + const timestamps = Object.keys(intervals).map((key) => intervals[key].timestamp); + expect(timestamps).toEqual([ + '1601856270140', + '1601852670140', + '1601849070140', + '1601845470140', + '1601841870140', + '1601838270140', + '1601834670140', + '1601831070140', + '1601827470140', + '1601823870140', + '1601820270140', + '1601816670140', + '1601813070140', + '1601809470140', + '1601805870140', + '1601802270140', + '1601798670140', + '1601795070140', + '1601791470140', + '1601787870140', + '1601784270140', + '1601780670140', + '1601777070140', + '1601773470140', + '1601769870140', + ]); + }); + + test('returns error if range is anything other than "h" or "d"', () => { + expect(() => getInterval('m', 1601856270140)).toThrow(); + }); + }); + + describe('getSequenceAggs', () => { + test('it aggregates events by sequences', () => { + const mockResponse = getMockSequenceResponse(); + const sequenceAggs = getSequenceAggs( + mockResponse, + 'h', + '2020-10-04T15:00:00.368707900Z', + '2020-10-04T16:00:00.368707900Z' + ); + + expect(sequenceAggs.data).toEqual([ + { g: 'Seq. 1', x: '2020-10-04T15:16:54.368707900Z', y: 1 }, + { g: 'Seq. 1', x: '2020-10-04T15:50:54.368707900Z', y: 1 }, + { g: 'Seq. 2', x: '2020-10-04T15:06:54.368707900Z', y: 1 }, + { g: 'Seq. 2', x: '2020-10-04T15:15:54.368707900Z', y: 1 }, + ]); + }); + }); +}); diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/helpers.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/helpers.ts new file mode 100644 index 00000000000000..0b2eba33b93d66 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/eql/helpers.ts @@ -0,0 +1,190 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import moment from 'moment'; +import { Unit } from '@elastic/datemath'; + +import * as i18n from '../../../detections/components/rules/query_preview/translations'; +import { EqlSearchStrategyResponse } from '../../../../../data_enhanced/common'; +import { InspectResponse } from '../../../types'; +import { EqlPreviewResponse, Source } from './types'; +import { EqlSearchResponse } from '../../../../common/detection_engine/types'; +import { HITS_THRESHOLD } from '../../../detections/components/rules/query_preview/helpers'; + +type EqlAggBuckets = Record; + +export const EQL_QUERY_EVENT_SIZE = 100; + +// Calculates which 2 min bucket segment, event should be +// sorted into +export const calculateBucketForHour = (eventTimestamp: number, relativeNow: number): number => { + const diff: number = relativeNow - eventTimestamp; + const minutes: number = Math.floor(diff / 60000); + return Math.ceil(minutes / 2) * 2; +}; + +// Calculates which 1 hour bucket segment, event should be +// sorted into +export const calculateBucketForDay = (eventTimestamp: number, relativeNow: number): number => { + const diff: number = relativeNow - eventTimestamp; + const minutes: number = Math.floor(diff / 60000); + return Math.ceil(minutes / 60); +}; + +export const constructWarnings = (timestampIssue: boolean, hits: number, range: Unit): string[] => { + let warnings: string[] = []; + + if (timestampIssue) { + warnings = [i18n.PREVIEW_WARNING_TIMESTAMP]; + } + + if (hits === EQL_QUERY_EVENT_SIZE) { + warnings = [...warnings, i18n.PREVIEW_WARNING_CAP_HIT(EQL_QUERY_EVENT_SIZE)]; + } + + if (hits > HITS_THRESHOLD[range]) { + warnings = [...warnings, i18n.QUERY_PREVIEW_NOISE_WARNING]; + } + + return warnings; +}; + +export const formatInspect = ( + response: EqlSearchStrategyResponse> +): InspectResponse => { + if (response != null) { + return { + dsl: [JSON.stringify(response.rawResponse.meta.request.params, null, 2)] ?? [], + response: [JSON.stringify(response.rawResponse.body, null, 2)] ?? [], + }; + } + + return { + dsl: [], + response: [], + }; +}; + +// NOTE: Eql does not support aggregations, this is an in-memory +// hand-spun aggregation for the events to give the user a visual +// representation of their query results +export const getEqlAggsData = ( + response: EqlSearchStrategyResponse>, + range: Unit, + to: string, + from: string +): EqlPreviewResponse => { + const { dsl, response: inspectResponse } = formatInspect(response); + // The upper bound of the timestamps + const relativeNow: number = Date.parse(from); + const accumulator: EqlAggBuckets = getInterval(range, relativeNow); + const events = response.rawResponse.body.hits.events ?? []; + const totalCount = response.rawResponse.body.hits.total.value; + let timestampNotFound = false; + + const buckets = events.reduce((acc, hit) => { + const timestamp = hit._source['@timestamp']; + if (timestamp == null) { + timestampNotFound = true; + return acc; + } + + const eventTimestamp: number = Date.parse(timestamp); + const bucket = + range === 'h' + ? calculateBucketForHour(eventTimestamp, relativeNow) + : calculateBucketForDay(eventTimestamp, relativeNow); + if (acc[bucket] != null) { + acc[bucket].total += 1; + } + return acc; + }, accumulator); + const data = Object.keys(buckets).map((key) => { + return { x: Number(buckets[key].timestamp), y: buckets[key].total, g: 'hits' }; + }); + + const isAllZeros = data.every(({ y }) => y === 0); + + const warnings = constructWarnings(timestampNotFound, totalCount, range); + + return { + data, + totalCount: isAllZeros ? 0 : totalCount, + lte: from, + gte: to, + inspect: { + dsl, + response: inspectResponse, + }, + warnings, + }; +}; + +export const createIntervalArray = (start: number, end: number, multiplier: number) => { + return Array(end - start + 1) + .fill(0) + .map((_, idx) => start + idx * multiplier); +}; + +export const getInterval = (range: Unit, relativeNow: number): EqlAggBuckets => { + switch (range) { + case 'h': + return createIntervalArray(0, 30, 2).reduce((acc, int) => { + return { + ...acc, + [int]: { timestamp: moment(relativeNow).subtract(int, 'm').format('x'), total: 0 }, + }; + }, {}); + case 'd': + return createIntervalArray(0, 24, 1).reduce((acc, int) => { + return { + ...acc, + [int]: { timestamp: moment(relativeNow).subtract(int, 'h').format('x'), total: 0 }, + }; + }, {}); + default: + throw new Error('Invalid time range selected'); + } +}; + +export const getSequenceAggs = ( + response: EqlSearchStrategyResponse>, + range: Unit, + to: string, + from: string +): EqlPreviewResponse => { + const { dsl, response: inspectResponse } = formatInspect(response); + const sequences = response.rawResponse.body.hits.sequences ?? []; + const totalCount = response.rawResponse.body.hits.total.value; + let timestampNotFound = false; + + const data = sequences.map((sequence, i) => { + return sequence.events.map((seqEvent) => { + if (seqEvent._source['@timestamp'] == null) { + timestampNotFound = true; + return {}; + } + return { + x: seqEvent._source['@timestamp'], + y: 1, + g: `Seq. ${i + 1}`, + }; + }); + }); + + const warnings = constructWarnings(timestampNotFound, totalCount, range); + + return { + data: data.flat(), + totalCount, + lte: from, + gte: to, + inspect: { + dsl, + response: inspectResponse, + }, + warnings, + }; +}; diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/index.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/index.ts new file mode 100644 index 00000000000000..1f3f40b5462c73 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/eql/index.ts @@ -0,0 +1,7 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export { useEqlPreview } from './use_eql_preview'; diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/types.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/types.ts new file mode 100644 index 00000000000000..e7ccf83591d815 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/eql/types.ts @@ -0,0 +1,20 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { InspectResponse } from '../../../types'; +import { ChartData } from '../../components/charts/common'; + +export interface EqlPreviewResponse { + data: ChartData[]; + totalCount: number; + lte: string; + gte: string; + inspect: InspectResponse; + warnings: string[]; +} + +export interface Source { + '@timestamp': string; +} diff --git a/x-pack/plugins/security_solution/public/common/hooks/eql/use_eql_preview.ts b/x-pack/plugins/security_solution/public/common/hooks/eql/use_eql_preview.ts new file mode 100644 index 00000000000000..384395b34e62b3 --- /dev/null +++ b/x-pack/plugins/security_solution/public/common/hooks/eql/use_eql_preview.ts @@ -0,0 +1,12 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { useAsync, withOptionalSignal } from '../../../shared_imports'; +import { getEqlPreview } from './api'; + +const getEqlPreviewWithOptionalSignal = withOptionalSignal(getEqlPreview); + +export const useEqlPreview = () => useAsync(getEqlPreviewWithOptionalSignal); diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/eql_histogram.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/eql_histogram.tsx new file mode 100644 index 00000000000000..3211afea821b09 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/eql_histogram.tsx @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import React, { useEffect, useMemo } from 'react'; +import { EuiFlexGroup, EuiFlexItem, EuiSpacer, EuiText } from '@elastic/eui'; + +import * as i18n from './translations'; +import { BarChart } from '../../../../common/components/charts/barchart'; +import { getHistogramConfig } from './helpers'; +import { ChartData, ChartSeriesConfigs } from '../../../../common/components/charts/common'; +import { InspectQuery } from '../../../../common/store/inputs/model'; +import { useGlobalTime } from '../../../../common/containers/use_global_time'; +import { Panel } from '../../../../common/components/panel'; +import { HeaderSection } from '../../../../common/components/header_section'; + +export const ID = 'queryEqlPreviewHistogramQuery'; + +interface PreviewEqlQueryHistogramProps { + to: string; + from: string; + totalHits: number; + data: ChartData[]; + inspect: InspectQuery; +} + +export const PreviewEqlQueryHistogram = ({ + from, + to, + totalHits, + data, + inspect, +}: PreviewEqlQueryHistogramProps) => { + const { setQuery, isInitializing } = useGlobalTime(); + + useEffect((): void => { + if (!isInitializing) { + setQuery({ id: ID, inspect, loading: false, refetch: () => {} }); + } + }, [setQuery, inspect, isInitializing]); + + const barConfig = useMemo((): ChartSeriesConfigs => getHistogramConfig(to, from), [from, to]); + + return ( + <> + + + + + + + + + + <> + + +

{i18n.PREVIEW_QUERY_DISCLAIMER_EQL}

+
+ +
+
+
+ + ); +}; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/helpers.ts b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/helpers.ts new file mode 100644 index 00000000000000..4cf37236510d73 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/helpers.ts @@ -0,0 +1,109 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Position, ScaleType } from '@elastic/charts'; +import { EuiSelectOption } from '@elastic/eui'; + +import * as i18n from './translations'; +import { histogramDateTimeFormatter } from '../../../../common/components/utils'; +import { ChartSeriesConfigs } from '../../../../common/components/charts/common'; +import { Type } from '../../../../../common/detection_engine/schemas/common/schemas'; + +export const HITS_THRESHOLD: Record = { + h: 1, + d: 24, + M: 730, +}; + +export const getTimeframeOptions = (ruleType: Type): EuiSelectOption[] => { + if (ruleType === 'eql') { + return [ + { value: 'h', text: 'Last hour' }, + { value: 'd', text: 'Last day' }, + ]; + } else { + return [ + { value: 'h', text: 'Last hour' }, + { value: 'd', text: 'Last day' }, + { value: 'M', text: 'Last month' }, + ]; + } +}; + +export const getHistogramConfig = (to: string, from: string): ChartSeriesConfigs => { + return { + series: { + xScaleType: ScaleType.Time, + yScaleType: ScaleType.Linear, + stackAccessors: ['g'], + }, + axis: { + xTickFormatter: histogramDateTimeFormatter([to, from]), + yTickFormatter: (value: string | number): string => value.toLocaleString(), + tickSize: 8, + }, + yAxisTitle: i18n.QUERY_GRAPH_COUNT, + settings: { + legendPosition: Position.Right, + showLegend: true, + showLegendExtra: true, + theme: { + scales: { + barsPadding: 0.08, + }, + chartMargins: { + left: 0, + right: 0, + top: 0, + bottom: 0, + }, + chartPaddings: { + left: 0, + right: 0, + top: 0, + bottom: 0, + }, + }, + }, + customHeight: 200, + }; +}; + +export const getThresholdHistogramConfig = (height: number | undefined): ChartSeriesConfigs => { + return { + series: { + xScaleType: ScaleType.Linear, + yScaleType: ScaleType.Linear, + stackAccessors: ['g'], + }, + axis: { + tickSize: 8, + }, + yAxisTitle: i18n.QUERY_GRAPH_COUNT, + settings: { + legendPosition: Position.Right, + showLegend: true, + showLegendExtra: true, + theme: { + scales: { + barsPadding: 0.08, + }, + chartMargins: { + left: 0, + right: 0, + top: 0, + bottom: 0, + }, + chartPaddings: { + left: 0, + right: 0, + top: 0, + bottom: 0, + }, + }, + }, + customHeight: height ?? 200, + }; +}; diff --git a/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx new file mode 100644 index 00000000000000..43dcdb7b7d58d0 --- /dev/null +++ b/x-pack/plugins/security_solution/public/detections/components/rules/query_preview/index.tsx @@ -0,0 +1,258 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import React, { useCallback, useState, useMemo, useEffect } from 'react'; +import { Unit } from '@elastic/datemath'; +import { getOr } from 'lodash/fp'; +import styled from 'styled-components'; +import { + EuiFlexGroup, + EuiFlexItem, + EuiSelect, + EuiFormRow, + EuiButton, + EuiCallOut, + EuiSelectOption, + EuiText, + EuiSpacer, +} from '@elastic/eui'; + +import * as i18n from './translations'; +import { useKibana } from '../../../../common/lib/kibana'; +import { ESQueryStringQuery } from '../../../../../common/typed_json'; +import { getQueryFilter } from '../../../../../common/detection_engine/get_query_filter'; +import { FieldValueQueryBar } from '../query_bar'; +import { Filter } from '../../../../../../../../src/plugins/data/common/es_query'; +import { Language, Type } from '../../../../../common/detection_engine/schemas/common/schemas'; +import { PreviewEqlQueryHistogram } from './eql_histogram'; +import { useEqlPreview } from '../../../../common/hooks/eql'; +import { useAppToasts } from '../../../../common/hooks/use_app_toasts'; +import { PreviewNonEqlQueryHistogram } from './non_eql_histogram'; +import { getTimeframeOptions } from './helpers'; +import { PreviewThresholdQueryHistogram } from './threshold_histogram'; +import { formatDate } from '../../../../common/components/super_date_picker'; + +const Select = styled(EuiSelect)` + width: ${({ theme }) => theme.eui.euiSuperDatePickerWidth}; +`; + +const PreviewButton = styled(EuiButton)` + margin-left: 0; +`; + +interface PreviewQueryProps { + dataTestSubj: string; + idAria: string; + query: FieldValueQueryBar | undefined; + index: string[]; + ruleType: Type; + threshold: { field: string | undefined; value: number } | undefined; + isDisabled: boolean; +} + +export const PreviewQuery = ({ + ruleType, + dataTestSubj, + idAria, + query, + index, + threshold, + isDisabled, +}: PreviewQueryProps) => { + const { data } = useKibana().services; + const { addError } = useAppToasts(); + + const [timeframeOptions, setTimeframeOptions] = useState([]); + const [showHistogram, setShowHistogram] = useState(false); + const [timeframe, setTimeframe] = useState('h'); + const [warnings, setWarnings] = useState([]); + const [queryFilter, setQueryFilter] = useState(undefined); + const [toTime, setTo] = useState(''); + const [fromTime, setFrom] = useState(''); + const { + error: eqlError, + start: startEql, + result: eqlQueryResult, + loading: eqlQueryLoading, + } = useEqlPreview(); + + const queryString = useMemo((): string => getOr('', 'query.query', query), [query]); + const language = useMemo((): Language => getOr('kuery', 'query.language', query), [query]); + const filters = useMemo((): Filter[] => (query != null ? query.filters : []), [query]); + + const handleCalculateTimeRange = useCallback((): void => { + const from = formatDate('now'); + const to = formatDate(`now-1${timeframe}`); + + setTo(to); + setFrom(from); + }, [timeframe]); + + const handlePreviewEqlQuery = useCallback((): void => { + startEql({ + data, + index, + query: queryString, + fromTime, + toTime, + interval: timeframe, + }); + }, [startEql, data, index, queryString, fromTime, toTime, timeframe]); + + const handleSelectPreviewTimeframe = ({ + target: { value }, + }: React.ChangeEvent): void => { + setTimeframe(value as Unit); + setShowHistogram(false); + }; + + const handlePreviewClicked = useCallback((): void => { + handleCalculateTimeRange(); + + if (ruleType === 'eql') { + setShowHistogram(true); + handlePreviewEqlQuery(); + } else { + const builtFilterQuery = { + ...((getQueryFilter( + queryString, + language, + filters, + index, + [], + true + ) as unknown) as ESQueryStringQuery), + }; + if (builtFilterQuery != null) { + setShowHistogram(true); + } + setQueryFilter(builtFilterQuery); + } + }, [ + filters, + handleCalculateTimeRange, + handlePreviewEqlQuery, + index, + language, + queryString, + ruleType, + ]); + + useEffect((): void => { + if (eqlError != null) { + addError(eqlError, { title: i18n.PREVIEW_QUERY_ERROR }); + } + }, [eqlError, addError]); + + // reset when rule type changes + useEffect((): void => { + const options = getTimeframeOptions(ruleType); + + setShowHistogram(false); + setTimeframe('h'); + setTimeframeOptions(options); + setWarnings([]); + }, [ruleType]); + + // reset when timeframe or query changes + useEffect((): void => { + setShowHistogram(false); + setWarnings([]); + }, [timeframe, queryString]); + + useEffect((): void => { + if (eqlQueryResult != null) { + setWarnings((prevWarnings) => { + if (eqlQueryResult.warnings.join() !== prevWarnings.join()) { + return eqlQueryResult.warnings; + } + + return prevWarnings; + }); + } + }, [eqlQueryResult]); + + const thresholdFieldExists = useMemo( + (): boolean => threshold != null && threshold.field != null && threshold.field.trim() !== '', + [threshold] + ); + + const showNonEqlHistogram = useMemo((): boolean => { + return ( + ruleType === 'query' || + ruleType === 'saved_query' || + (ruleType === 'threshold' && !thresholdFieldExists) + ); + }, [ruleType, thresholdFieldExists]); + + return ( + <> + + + +