From 8424a925332d5bd7d93148fbab72d1f518933485 Mon Sep 17 00:00:00 2001 From: Alison Goryachev Date: Mon, 14 Jun 2021 10:50:20 -0400 Subject: [PATCH 01/77] [ILM] Migrate to new page layout (#101927) --- .../__snapshots__/policy_table.test.tsx.snap | 98 +++-- .../edit_policy/edit_policy.container.tsx | 70 ++-- .../sections/edit_policy/edit_policy.tsx | 351 +++++++++--------- .../policy_table/policy_table.container.tsx | 70 ++-- .../sections/policy_table/policy_table.tsx | 101 +++-- 5 files changed, 339 insertions(+), 351 deletions(-) diff --git a/x-pack/plugins/index_lifecycle_management/__jest__/__snapshots__/policy_table.test.tsx.snap b/x-pack/plugins/index_lifecycle_management/__jest__/__snapshots__/policy_table.test.tsx.snap index 8d839e196916b6..556ac35d0565e8 100644 --- a/x-pack/plugins/index_lifecycle_management/__jest__/__snapshots__/policy_table.test.tsx.snap +++ b/x-pack/plugins/index_lifecycle_management/__jest__/__snapshots__/policy_table.test.tsx.snap @@ -46,71 +46,67 @@ Array [ `; exports[`policy table should show empty state when there are not any policies 1`] = ` -
+
+ - -
- -

- Create your first index lifecycle policy -

-
-
-

- An index lifecycle policy helps you manage your indices as they age. -

-
- + Create your first index lifecycle policy +
-
+ +
+
+ -
+ +
-
+ `; exports[`policy table should sort when linked indices header is clicked 1`] = ` diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.container.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.container.tsx index fe82b200930724..07c2228863b810 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.container.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.container.tsx @@ -7,7 +7,7 @@ import React, { useEffect } from 'react'; import { RouteComponentProps } from 'react-router-dom'; -import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui'; +import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner, EuiPageContent } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { MIN_SEARCHABLE_SNAPSHOT_LICENSE } from '../../../../common/constants'; @@ -52,43 +52,47 @@ export const EditPolicy: React.FunctionComponent} - body={ - - } - /> + + } + body={ + + } + /> + ); } if (error || !policies) { const { statusCode, message } = error ? error : { statusCode: '', message: '' }; return ( - - - - } - body={ -

- {message} ({statusCode}) -

- } - actions={ - - - - } - /> + + + + + } + body={ +

+ {message} ({statusCode}) +

+ } + actions={ + + + + } + /> + ); } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx index d7368249d76e8c..172e8259b87af3 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/edit_policy/edit_policy.tsx @@ -19,15 +19,10 @@ import { EuiFlexItem, EuiFormRow, EuiHorizontalRule, - EuiPage, - EuiPageBody, - EuiPageContent, - EuiPageContentHeader, - EuiPageContentHeaderSection, EuiSpacer, EuiSwitch, EuiText, - EuiTitle, + EuiPageHeader, } from '@elastic/eui'; import { TextField, useForm, useFormData, useKibana } from '../../../shared_imports'; @@ -153,201 +148,199 @@ export const EditPolicy: React.FunctionComponent = ({ history }) => { }; return ( - - - - - - -

- {isNewPolicy - ? i18n.translate('xpack.indexLifecycleMgmt.editPolicy.createPolicyMessage', { - defaultMessage: 'Create policy', - }) - : i18n.translate('xpack.indexLifecycleMgmt.editPolicy.editPolicyMessage', { - defaultMessage: 'Edit policy {originalPolicyName}', - values: { originalPolicyName }, - })} -

- - - - + <> + + {isNewPolicy + ? i18n.translate('xpack.indexLifecycleMgmt.editPolicy.createPolicyMessage', { + defaultMessage: 'Create policy', + }) + : i18n.translate('xpack.indexLifecycleMgmt.editPolicy.editPolicyMessage', { + defaultMessage: 'Edit policy {originalPolicyName}', + values: { originalPolicyName }, + })} + + } + bottomBorder + rightSideItems={[ + + + , + ]} + /> + + + +
+ {isNewPolicy ? null : ( + + +

+ + + + .{' '} - - - - - {isNewPolicy ? null : ( - - -

- - - - .{' '} - -

- - - - - { - setSaveAsNew(e.target.checked); - }} - label={ - - - - } - /> - - - )} - - {saveAsNew || isNewPolicy ? ( - - path={policyNamePath} - config={{ - label: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.policyNameLabel', { - defaultMessage: 'Policy name', - }), - helpText: i18n.translate( - 'xpack.indexLifecycleMgmt.editPolicy.validPolicyNameMessage', - { - defaultMessage: - 'A policy name cannot start with an underscore and cannot contain a comma or a space.', - } - ), - validations: policyNameValidations, - }} - component={TextField} - componentProps={{ - fullWidth: false, - euiFieldProps: { - 'data-test-subj': 'policyNameField', - }, + /> +

+ + + + + { + setSaveAsNew(e.target.checked); }} + label={ + + + + } /> - ) : null} - - - - - - - -
- - - - - + + + )} + + {saveAsNew || isNewPolicy ? ( + + path={policyNamePath} + config={{ + label: i18n.translate('xpack.indexLifecycleMgmt.editPolicy.policyNameLabel', { + defaultMessage: 'Policy name', + }), + helpText: i18n.translate( + 'xpack.indexLifecycleMgmt.editPolicy.validPolicyNameMessage', + { + defaultMessage: + 'A policy name cannot start with an underscore and cannot contain a comma or a space.', + } + ), + validations: policyNameValidations, + }} + component={TextField} + componentProps={{ + fullWidth: false, + euiFieldProps: { + 'data-test-subj': 'policyNameField', + }, + }} + /> + ) : null} + + + + + + + +
+ + + + + + + + + {isAllowedByLicense && ( + <> - + + + )} - {isAllowedByLicense && ( - <> - - - - )} - - {/* We can't add the here as it breaks the layout + {/* We can't add the here as it breaks the layout and makes the connecting line go further that it needs to. There is an issue in EUI to fix this (https://github.com/elastic/eui/issues/4492) */} - -
+ +
- + - - - - - - - - {saveAsNew ? ( - - ) : ( - - )} - - - - - - - - - - + + + + - - {isShowingPolicyJsonFlyout ? ( + + {saveAsNew ? ( ) : ( )} + + + + + + + - {isShowingPolicyJsonFlyout ? ( - setIsShowingPolicyJsonFlyout(false)} - /> - ) : null} - -
- - + + + {isShowingPolicyJsonFlyout ? ( + + ) : ( + + )} + + + + + {isShowingPolicyJsonFlyout ? ( + setIsShowingPolicyJsonFlyout(false)} + /> + ) : null} + + ); }; diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.container.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.container.tsx index 7cf50de0ee9997..deac2bb239d30a 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.container.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.container.tsx @@ -8,7 +8,7 @@ import React, { useEffect } from 'react'; import { ApplicationStart } from 'kibana/public'; import { RouteComponentProps } from 'react-router-dom'; -import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner } from '@elastic/eui'; +import { EuiButton, EuiEmptyPrompt, EuiLoadingSpinner, EuiPageContent } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import { PolicyTable as PresentationComponent } from './policy_table'; import { useKibana } from '../../../shared_imports'; @@ -33,43 +33,47 @@ export const PolicyTable: React.FunctionComponent = if (isLoading) { return ( - } - body={ - - } - /> + + } + body={ + + } + /> + ); } if (error) { const { statusCode, message } = error ? error : { statusCode: '', message: '' }; return ( - - - - } - body={ -

- {message} ({statusCode}) -

- } - actions={ - - - - } - /> + + + + + } + body={ +

+ {message} ({statusCode}) +

+ } + actions={ + + + + } + /> + ); } diff --git a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx index 3b0f815800ba3d..ba89d6c895d93e 100644 --- a/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx +++ b/x-pack/plugins/index_lifecycle_management/public/application/sections/policy_table/policy_table.tsx @@ -16,9 +16,7 @@ import { EuiFlexGroup, EuiFlexItem, EuiSpacer, - EuiTitle, - EuiText, - EuiPageBody, + EuiPageHeader, EuiPageContent, } from '@elastic/eui'; import { ApplicationStart } from 'kibana/public'; @@ -119,67 +117,60 @@ export const PolicyTable: React.FunctionComponent = ({ ); } else { return ( - - - + + + + + } + body={ + +

- - } - body={ - -

- -

- - } - actions={createPolicyButton} - /> - - +

+ + } + actions={createPolicyButton} + /> + ); } return ( - - - {confirmModal} + <> + {confirmModal} - - - -

- -

- - - {createPolicyButton} - - - -

+ -

- + + } + description={ + + } + bottomBorder + rightSideItems={[createPolicyButton]} + /> - - {content} - - + + + {content} + ); }; From 277212df0b919b57d3dadd40bf059eb908b44fd0 Mon Sep 17 00:00:00 2001 From: Stacey Gammon Date: Mon, 14 Jun 2021 11:04:27 -0400 Subject: [PATCH 02/77] Update building_a_plugin.mdx (#101921) * Update building_a_plugin.mdx * put back the numbers on the same line * Add info about requiredBundles * Fix numbers again * Update dev_docs/tutorials/building_a_plugin.mdx Co-authored-by: Mikhail Shustov * Update dev_docs/tutorials/building_a_plugin.mdx Co-authored-by: Mikhail Shustov Co-authored-by: Mikhail Shustov Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- dev_docs/tutorials/building_a_plugin.mdx | 79 ++++++++++++++++++------ 1 file changed, 59 insertions(+), 20 deletions(-) diff --git a/dev_docs/tutorials/building_a_plugin.mdx b/dev_docs/tutorials/building_a_plugin.mdx index cee5a9a399de5b..e751ce7d01b165 100644 --- a/dev_docs/tutorials/building_a_plugin.mdx +++ b/dev_docs/tutorials/building_a_plugin.mdx @@ -4,7 +4,7 @@ slug: /kibana-dev-docs/tutorials/build-a-plugin title: Kibana plugin tutorial summary: Anatomy of a Kibana plugin and how to build one date: 2021-02-05 -tags: ['kibana','onboarding', 'dev', 'tutorials'] +tags: ['kibana', 'onboarding', 'dev', 'tutorials'] --- Prereading material: @@ -14,7 +14,7 @@ Prereading material: ## The anatomy of a plugin Plugins are defined as classes and present themselves to Kibana through a simple wrapper function. A plugin can have browser-side code, server-side code, -or both. There is no architectural difference between a plugin in the browser and a plugin on the server. In both places, you describe your plugin similarly, +or both. There is no architectural difference between a plugin in the browser and a plugin on the server. In both places, you describe your plugin similarly, and you interact with Core and other plugins in the same way. The basic file structure of a Kibana plugin named demo that has both client-side and server-side code would be: @@ -33,7 +33,7 @@ plugins/ index.ts [6] ``` -### [1] kibana.json +### [1] kibana.json `kibana.json` is a static manifest file that is used to identify the plugin and to specify if this plugin has server-side code, browser-side code, or both: @@ -42,14 +42,33 @@ plugins/ "id": "demo", "version": "kibana", "server": true, - "ui": true + "ui": true, + "owner": { [1] + "name": "App Services", + "githubTeam": "kibana-app-services" + }, + "description": "This plugin extends Kibana by doing xyz, and allows other plugins to extend Kibana by offering abc functionality. It also exposes some helper utilities that do efg", [2] + "requiredPlugins": ["data"], [3] + "optionalPlugins": ["alerting"] [4] + "requiredBundles": ["anotherPlugin"] [5] } ``` +[1], [2]: Every internal plugin should fill in the owner and description properties. + +[3], [4]: Any plugin that you have a dependency on should be listed in `requiredPlugins` or `optionalPlugins`. Doing this will ensure that you have access to that plugin's start and setup contract inside your own plugin's start and setup lifecycle methods. If a plugin you optionally depend on is not installed or disabled, it will be undefined if you try to access it. If a plugin you require is not installed or disabled, kibana will fail to build. + +[5]: Don't worry too much about getting 5 right. The build optimizer will complain if any of these values are incorrect. + + + + You don't need to declare a dependency on a plugin if you only wish to access its types. + + ### [2] public/index.ts -`public/index.ts` is the entry point into the client-side code of this plugin. It must export a function named plugin, which will receive a standard set of - core capabilities as an argument. It should return an instance of its plugin class for Kibana to load. +`public/index.ts` is the entry point into the client-side code of this plugin. Everything exported from this file will be a part of the plugins . If the plugin only exists to export static utilities, consider using a package. Otherwise, this file must export a function named plugin, which will receive a standard set of +core capabilities as an argument. It should return an instance of its plugin class for Kibana to load. ``` import type { PluginInitializerContext } from 'kibana/server'; @@ -60,13 +79,32 @@ export function plugin(initializerContext: PluginInitializerContext) { } ``` + + +1. When possible, use + +``` +export type { AType } from '...'` +``` + +instead of + +``` +export { AType } from '...'`. +``` + +Using the non-`type` variation will increase the bundle size unnecessarily and may unwillingly provide access to the implementation of `AType` class. + +2. Don't use `export *` in these top level index.ts files + + + ### [3] public/plugin.ts `public/plugin.ts` is the client-side plugin definition itself. Technically speaking, it does not need to be a class or even a separate file from the entry - point, but all plugins at Elastic should be consistent in this way. +point, but all plugins at Elastic should be consistent in this way. - - ```ts +```ts import type { Plugin, PluginInitializerContext, CoreSetup, CoreStart } from 'kibana/server'; export class DemoPlugin implements Plugin { @@ -84,10 +122,9 @@ export class DemoPlugin implements Plugin { // called when plugin is torn down during Kibana's shutdown sequence } } - ``` - +``` -### [4] server/index.ts +### [4] server/index.ts `server/index.ts` is the entry-point into the server-side code of this plugin. It is identical in almost every way to the client-side entry-point: @@ -115,7 +152,7 @@ export class DemoPlugin implements Plugin { } ``` -Kibana does not impose any technical restrictions on how the the internals of a plugin are architected, though there are certain +Kibana does not impose any technical restrictions on how the the internals of a plugin are architected, though there are certain considerations related to how plugins integrate with core APIs and APIs exposed by other plugins that may greatly impact how they are built. ### [6] common/index.ts @@ -124,8 +161,8 @@ considerations related to how plugins integrate with core APIs and APIs exposed ## How plugin's interact with each other, and Core -The lifecycle-specific contracts exposed by core services are always passed as the first argument to the equivalent lifecycle function in a plugin. -For example, the core http service exposes a function createRouter to all plugin setup functions. To use this function to register an HTTP route handler, +The lifecycle-specific contracts exposed by core services are always passed as the first argument to the equivalent lifecycle function in a plugin. +For example, the core http service exposes a function createRouter to all plugin setup functions. To use this function to register an HTTP route handler, a plugin just accesses it off of the first argument: ```ts @@ -135,14 +172,16 @@ export class DemoPlugin { public setup(core: CoreSetup) { const router = core.http.createRouter(); // handler is called when '/path' resource is requested with `GET` method - router.get({ path: '/path', validate: false }, (context, req, res) => res.ok({ content: 'ok' })); + router.get({ path: '/path', validate: false }, (context, req, res) => + res.ok({ content: 'ok' }) + ); } } ``` Unlike core, capabilities exposed by plugins are not automatically injected into all plugins. Instead, if a plugin wishes to use the public interface provided by another plugin, it must first declare that plugin as a - dependency in it’s kibana.json manifest file. +dependency in it’s kibana.json manifest file. ** foobar plugin.ts: ** @@ -174,8 +213,8 @@ export class MyPlugin implements Plugin { } } ``` -[1] We highly encourage plugin authors to explicitly declare public interfaces for their plugins. +[1] We highly encourage plugin authors to explicitly declare public interfaces for their plugins. ** demo kibana.json** @@ -194,7 +233,7 @@ With that specified in the plugin manifest, the appropriate interfaces are then import type { CoreSetup, CoreStart } from 'kibana/server'; import type { FoobarPluginSetup, FoobarPluginStart } from '../../foobar/server'; -interface DemoSetupPlugins { [1] +interface DemoSetupPlugins { [1] foobar: FoobarPluginSetup; } @@ -218,7 +257,7 @@ export class DemoPlugin { public stop() {} } ``` - + [1] The interface for plugin’s dependencies must be manually composed. You can do this by importing the appropriate type from the plugin and constructing an interface where the property name is the plugin’s ID. [2] These manually constructed types should then be used to specify the type of the second argument to the plugin. From 0e7d4fed93b98d4b746f30141535a852c8b42f69 Mon Sep 17 00:00:00 2001 From: Josh Dover <1813008+joshdover@users.noreply.github.com> Date: Mon, 14 Jun 2021 17:14:02 +0200 Subject: [PATCH 03/77] Fix delayed status API updates in alerting and task_manager (#101778) --- test/functional/apps/bundles/index.js | 9 ++++++ .../server_integration/http/platform/cache.ts | 11 +++++++ .../alerting/server/health/get_state.test.ts | 20 +++++------- .../alerting/server/health/get_state.ts | 13 +++++++- x-pack/plugins/alerting/server/plugin.ts | 31 ++++++++++++------- x-pack/plugins/task_manager/server/plugin.ts | 14 ++++----- 6 files changed, 65 insertions(+), 33 deletions(-) diff --git a/test/functional/apps/bundles/index.js b/test/functional/apps/bundles/index.js index d13e74dd4eed95..577035a8c343cc 100644 --- a/test/functional/apps/bundles/index.js +++ b/test/functional/apps/bundles/index.js @@ -18,6 +18,15 @@ export default function ({ getService }) { let buildNum; before(async () => { + // Wait for status to become green + let status; + const start = Date.now(); + do { + const resp = await supertest.get('/api/status'); + status = resp.status; + // Stop polling once status stabilizes OR once 40s has passed + } while (status !== 200 && Date.now() - start < 40_000); + const resp = await supertest.get('/api/status').expect(200); buildNum = resp.body.version.build_number; }); diff --git a/test/server_integration/http/platform/cache.ts b/test/server_integration/http/platform/cache.ts index 2c1aa90e963e24..a33f916cf1c4b6 100644 --- a/test/server_integration/http/platform/cache.ts +++ b/test/server_integration/http/platform/cache.ts @@ -12,6 +12,17 @@ export default function ({ getService }: FtrProviderContext) { const supertest = getService('supertest'); describe('kibana server cache-control', () => { + before(async () => { + // Wait for status to become green + let status; + const start = Date.now(); + do { + const resp = await supertest.get('/api/status'); + status = resp.status; + // Stop polling once status stabilizes OR once 40s has passed + } while (status !== 200 && Date.now() - start < 40_000); + }); + it('properly marks responses as private, with directives to disable caching', async () => { await supertest .get('/api/status') diff --git a/x-pack/plugins/alerting/server/health/get_state.test.ts b/x-pack/plugins/alerting/server/health/get_state.test.ts index 96627e10fb3bdf..2dddf81e3b7667 100644 --- a/x-pack/plugins/alerting/server/health/get_state.test.ts +++ b/x-pack/plugins/alerting/server/health/get_state.test.ts @@ -58,7 +58,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { const mockTaskManager = taskManagerMock.createStart(); mockTaskManager.get.mockResolvedValue(getHealthCheckTask()); const pollInterval = 100; - const halfInterval = Math.floor(pollInterval / 2); getHealthStatusStream( mockTaskManager, @@ -77,16 +76,15 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { pollInterval ).subscribe(); - // shouldn't fire before poll interval passes + // should fire before poll interval passes // should fire once each poll interval - jest.advanceTimersByTime(halfInterval); - expect(mockTaskManager.get).toHaveBeenCalledTimes(0); - jest.advanceTimersByTime(halfInterval); expect(mockTaskManager.get).toHaveBeenCalledTimes(1); jest.advanceTimersByTime(pollInterval); expect(mockTaskManager.get).toHaveBeenCalledTimes(2); jest.advanceTimersByTime(pollInterval); expect(mockTaskManager.get).toHaveBeenCalledTimes(3); + jest.advanceTimersByTime(pollInterval); + expect(mockTaskManager.get).toHaveBeenCalledTimes(4); }); it('should retry on error', async () => { @@ -94,7 +92,6 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { mockTaskManager.get.mockRejectedValue(new Error('Failure')); const retryDelay = 10; const pollInterval = 100; - const halfInterval = Math.floor(pollInterval / 2); getHealthStatusStream( mockTaskManager, @@ -114,28 +111,27 @@ describe('getHealthServiceStatusWithRetryAndErrorHandling', () => { retryDelay ).subscribe(); - jest.advanceTimersByTime(halfInterval); - expect(mockTaskManager.get).toHaveBeenCalledTimes(0); - jest.advanceTimersByTime(halfInterval); expect(mockTaskManager.get).toHaveBeenCalledTimes(1); + jest.advanceTimersByTime(pollInterval); + expect(mockTaskManager.get).toHaveBeenCalledTimes(2); // Retry on failure let numTimesCalled = 1; for (let i = 0; i < MAX_RETRY_ATTEMPTS; i++) { await tick(); jest.advanceTimersByTime(retryDelay); - expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled++ + 1); + expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled++ + 2); } // Once we've exceeded max retries, should not try again await tick(); jest.advanceTimersByTime(retryDelay); - expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled); + expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled + 1); // Once another poll interval passes, should call fn again await tick(); jest.advanceTimersByTime(pollInterval - MAX_RETRY_ATTEMPTS * retryDelay); - expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled + 1); + expect(mockTaskManager.get).toHaveBeenCalledTimes(numTimesCalled + 2); }); it('should return healthy status when health status is "ok"', async () => { diff --git a/x-pack/plugins/alerting/server/health/get_state.ts b/x-pack/plugins/alerting/server/health/get_state.ts index 30099614ea42b3..255037d7015a2b 100644 --- a/x-pack/plugins/alerting/server/health/get_state.ts +++ b/x-pack/plugins/alerting/server/health/get_state.ts @@ -7,7 +7,7 @@ import { i18n } from '@kbn/i18n'; import { defer, of, interval, Observable, throwError, timer } from 'rxjs'; -import { catchError, mergeMap, retryWhen, switchMap } from 'rxjs/operators'; +import { catchError, mergeMap, retryWhen, startWith, switchMap } from 'rxjs/operators'; import { Logger, SavedObjectsServiceStart, @@ -121,6 +121,17 @@ export const getHealthStatusStream = ( retryDelay?: number ): Observable> => interval(healthStatusInterval ?? HEALTH_STATUS_INTERVAL).pipe( + // Emit an initial check + startWith( + getHealthServiceStatusWithRetryAndErrorHandling( + taskManager, + logger, + savedObjects, + config, + retryDelay + ) + ), + // On each interval do a new check switchMap(() => getHealthServiceStatusWithRetryAndErrorHandling( taskManager, diff --git a/x-pack/plugins/alerting/server/plugin.ts b/x-pack/plugins/alerting/server/plugin.ts index 769243b8feaf6a..41ae9f15d9af90 100644 --- a/x-pack/plugins/alerting/server/plugin.ts +++ b/x-pack/plugins/alerting/server/plugin.ts @@ -7,7 +7,7 @@ import type { PublicMethodsOf } from '@kbn/utility-types'; import { first, map, share } from 'rxjs/operators'; -import { Observable } from 'rxjs'; +import { BehaviorSubject, Observable } from 'rxjs'; import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { combineLatest } from 'rxjs'; import { SecurityPluginSetup, SecurityPluginStart } from '../../security/server'; @@ -34,6 +34,7 @@ import { StatusServiceSetup, ServiceStatus, SavedObjectsBulkGetObject, + ServiceStatusLevels, } from '../../../../src/core/server'; import type { AlertingRequestHandlerContext } from './types'; import { defineRoutes } from './routes'; @@ -226,17 +227,23 @@ export class AlertingPlugin { this.config ); + const serviceStatus$ = new BehaviorSubject({ + level: ServiceStatusLevels.unavailable, + summary: 'Alerting is initializing', + }); + core.status.set(serviceStatus$); + core.getStartServices().then(async ([coreStart, startPlugins]) => { - core.status.set( - combineLatest([ - core.status.derivedStatus$, - getHealthStatusStream( - startPlugins.taskManager, - this.logger, - coreStart.savedObjects, - this.config - ), - ]).pipe( + combineLatest([ + core.status.derivedStatus$, + getHealthStatusStream( + startPlugins.taskManager, + this.logger, + coreStart.savedObjects, + this.config + ), + ]) + .pipe( map(([derivedStatus, healthStatus]) => { if (healthStatus.level > derivedStatus.level) { return healthStatus as ServiceStatus; @@ -246,7 +253,7 @@ export class AlertingPlugin { }), share() ) - ); + .subscribe(serviceStatus$); }); initializeAlertingHealth(this.logger, plugins.taskManager, core.getStartServices()); diff --git a/x-pack/plugins/task_manager/server/plugin.ts b/x-pack/plugins/task_manager/server/plugin.ts index 51199da26ee7da..d3e251b751ef8c 100644 --- a/x-pack/plugins/task_manager/server/plugin.ts +++ b/x-pack/plugins/task_manager/server/plugin.ts @@ -87,15 +87,13 @@ export class TaskManagerPlugin this.config! ); - core.getStartServices().then(async () => { - core.status.set( - combineLatest([core.status.derivedStatus$, serviceStatus$]).pipe( - map(([derivedStatus, serviceStatus]) => - serviceStatus.level > derivedStatus.level ? serviceStatus : derivedStatus - ) + core.status.set( + combineLatest([core.status.derivedStatus$, serviceStatus$]).pipe( + map(([derivedStatus, serviceStatus]) => + serviceStatus.level > derivedStatus.level ? serviceStatus : derivedStatus ) - ); - }); + ) + ); return { index: this.config.index, From 970e9a037bcb098ec84d9a41c5b1e680ecc05f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=B8ren=20Louv-Jansen?= Date: Mon, 14 Jun 2021 17:14:15 +0200 Subject: [PATCH 04/77] [APM] Add AWS and Azure icons for additional services (#101901) --- .../shared/span_icon/get_span_icon.ts | 17 ++++- .../shared/span_icon/span_icon.stories.tsx | 73 +++++++++++++++++++ 2 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 x-pack/plugins/apm/public/components/shared/span_icon/span_icon.stories.tsx diff --git a/x-pack/plugins/apm/public/components/shared/span_icon/get_span_icon.ts b/x-pack/plugins/apm/public/components/shared/span_icon/get_span_icon.ts index bebfcba1a93b40..e2e1391a2f842d 100644 --- a/x-pack/plugins/apm/public/components/shared/span_icon/get_span_icon.ts +++ b/x-pack/plugins/apm/public/components/shared/span_icon/get_span_icon.ts @@ -6,7 +6,6 @@ */ import { maybe } from '../../../../common/utils/maybe'; -import awsIcon from './icons/aws.svg'; import cassandraIcon from './icons/cassandra.svg'; import databaseIcon from './icons/database.svg'; import defaultIcon from './icons/default.svg'; @@ -33,12 +32,14 @@ const defaultTypeIcons: { [key: string]: string } = { resource: globeIcon, }; -const typeIcons: { [key: string]: { [key: string]: string } } = { +export const typeIcons: { [type: string]: { [subType: string]: string } } = { aws: { - servicename: awsIcon, + servicename: 'logoAWS', }, db: { cassandra: cassandraIcon, + cosmosdb: 'logoAzure', + dynamodb: 'logoAWS', elasticsearch: elasticsearchIcon, mongodb: mongodbIcon, mysql: mysqlIcon, @@ -51,8 +52,18 @@ const typeIcons: { [key: string]: { [key: string]: string } } = { websocket: websocketIcon, }, messaging: { + azurequeue: 'logoAzure', + azureservicebus: 'logoAzure', jms: javaIcon, kafka: kafkaIcon, + sns: 'logoAWS', + sqs: 'logoAWS', + }, + storage: { + azureblob: 'logoAzure', + azurefile: 'logoAzure', + azuretable: 'logoAzure', + s3: 'logoAWS', }, template: { handlebars: handlebarsIcon, diff --git a/x-pack/plugins/apm/public/components/shared/span_icon/span_icon.stories.tsx b/x-pack/plugins/apm/public/components/shared/span_icon/span_icon.stories.tsx new file mode 100644 index 00000000000000..951d3e61f18460 --- /dev/null +++ b/x-pack/plugins/apm/public/components/shared/span_icon/span_icon.stories.tsx @@ -0,0 +1,73 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + EuiFlexGrid, + EuiFlexItem, + EuiCopy, + EuiPanel, + EuiSpacer, + EuiCodeBlock, +} from '@elastic/eui'; +import React from 'react'; +import { storiesOf } from '@storybook/react'; +import { EuiThemeProvider } from '../../../../../../../src/plugins/kibana_react/common'; +import { SpanIcon } from './index'; +import { typeIcons } from './get_span_icon'; + +const types = Object.keys(typeIcons); + +storiesOf('shared/span_icon/span_icon', module) + .addDecorator((storyFn) => {storyFn()}) + .add( + 'Span icon', + () => { + return ( + <> + + {''} + + + + + {types.map((type) => { + const subTypes = Object.keys(typeIcons[type]); + return ( + <> + {subTypes.map((subType) => { + const id = `${type}.${subType}`; + return ( + + + {(copy) => ( + +  {' '} + {id} + + )} + + + ); + })} + + ); + })} + + + ); + }, + {} + ); From adda72edd28abf21b201477aaddc55b51e197596 Mon Sep 17 00:00:00 2001 From: Larry Gregory Date: Mon, 14 Jun 2021 12:23:28 -0400 Subject: [PATCH 05/77] Document platform security plugins in kibana.json (#101965) Co-authored-by: Kibana Machine <42973632+kibanamachine@users.noreply.github.com> --- src/plugins/security_oss/kibana.json | 5 +++++ src/plugins/spaces_oss/kibana.json | 5 +++++ x-pack/plugins/encrypted_saved_objects/kibana.json | 5 +++++ x-pack/plugins/security/kibana.json | 5 +++++ x-pack/plugins/spaces/kibana.json | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/src/plugins/security_oss/kibana.json b/src/plugins/security_oss/kibana.json index 70e37d586f1db3..c93b5c3b60714d 100644 --- a/src/plugins/security_oss/kibana.json +++ b/src/plugins/security_oss/kibana.json @@ -1,5 +1,10 @@ { "id": "securityOss", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin exposes a limited set of security functionality to OSS plugins.", "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["security"], diff --git a/src/plugins/spaces_oss/kibana.json b/src/plugins/spaces_oss/kibana.json index e048fb7ffb79c6..10127634618f1a 100644 --- a/src/plugins/spaces_oss/kibana.json +++ b/src/plugins/spaces_oss/kibana.json @@ -1,5 +1,10 @@ { "id": "spacesOss", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin exposes a limited set of spaces functionality to OSS plugins.", "version": "kibana", "server": false, "ui": true, diff --git a/x-pack/plugins/encrypted_saved_objects/kibana.json b/x-pack/plugins/encrypted_saved_objects/kibana.json index 74f797ba36a11f..4812afe8c20725 100644 --- a/x-pack/plugins/encrypted_saved_objects/kibana.json +++ b/x-pack/plugins/encrypted_saved_objects/kibana.json @@ -1,5 +1,10 @@ { "id": "encryptedSavedObjects", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin provides encryption and decryption utilities for saved objects containing sensitive information.", "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "encryptedSavedObjects"], diff --git a/x-pack/plugins/security/kibana.json b/x-pack/plugins/security/kibana.json index f6e7b8bf46a394..a29c01b0f31ccd 100644 --- a/x-pack/plugins/security/kibana.json +++ b/x-pack/plugins/security/kibana.json @@ -1,5 +1,10 @@ { "id": "security", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin provides authentication and authorization features, and exposes functionality to understand the capabilities of the currently authenticated user.", "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "security"], diff --git a/x-pack/plugins/spaces/kibana.json b/x-pack/plugins/spaces/kibana.json index e67931d4a3b8d9..673055b24e8b61 100644 --- a/x-pack/plugins/spaces/kibana.json +++ b/x-pack/plugins/spaces/kibana.json @@ -1,5 +1,10 @@ { "id": "spaces", + "owner": { + "name": "Platform Security", + "githubTeam": "kibana-security" + }, + "description": "This plugin provides the Spaces feature, which allows saved objects to be organized into meaningful categories.", "version": "8.0.0", "kibanaVersion": "kibana", "configPath": ["xpack", "spaces"], From 91b804f505da9614dc6b85cd64ae64f6d06bea0f Mon Sep 17 00:00:00 2001 From: Nicolas Chaulet Date: Mon, 14 Jun 2021 12:33:46 -0400 Subject: [PATCH 06/77] [Fleet] Display NOTICE.txt from package if it exists (#101663) --- .../plugins/fleet/common/types/models/epm.ts | 8 +- .../applications/integrations/constants.tsx | 6 +- .../integrations/sections/epm/constants.tsx | 7 +- .../epm/screens/detail/overview/details.tsx | 57 +++++++++---- .../screens/detail/overview/notice_modal.tsx | 79 +++++++++++++++++++ .../server/services/epm/archive/index.ts | 7 ++ .../fleet/server/services/epm/packages/get.ts | 1 + .../server/services/epm/registry/index.ts | 12 +++ 8 files changed, 155 insertions(+), 22 deletions(-) create mode 100644 x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/notice_modal.tsx diff --git a/x-pack/plugins/fleet/common/types/models/epm.ts b/x-pack/plugins/fleet/common/types/models/epm.ts index 0ef9f8b7ace36a..f19684b0445e2a 100644 --- a/x-pack/plugins/fleet/common/types/models/epm.ts +++ b/x-pack/plugins/fleet/common/types/models/epm.ts @@ -48,7 +48,12 @@ export type EpmPackageInstallStatus = 'installed' | 'installing'; export type DetailViewPanelName = 'overview' | 'policies' | 'settings' | 'custom'; export type ServiceName = 'kibana' | 'elasticsearch'; export type AgentAssetType = typeof agentAssetTypes; -export type AssetType = KibanaAssetType | ElasticsearchAssetType | ValueOf; +export type DocAssetType = 'doc' | 'notice'; +export type AssetType = + | KibanaAssetType + | ElasticsearchAssetType + | ValueOf + | DocAssetType; /* Enum mapping of a saved object asset type to how it would appear in a package file path (snake cased) @@ -344,6 +349,7 @@ export interface EpmPackageAdditions { latestVersion: string; assets: AssetsGroupedByServiceByType; removable?: boolean; + notice?: string; } type Merge = Omit> & diff --git a/x-pack/plugins/fleet/public/applications/integrations/constants.tsx b/x-pack/plugins/fleet/public/applications/integrations/constants.tsx index 403a47f4b94b2f..08197e18fec026 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/constants.tsx @@ -7,7 +7,7 @@ import type { IconType } from '@elastic/eui'; -import type { AssetType, ServiceName } from '../../types'; +import type { ServiceName } from '../../types'; import { ElasticsearchAssetType, KibanaAssetType } from '../../types'; export * from '../../constants'; @@ -20,8 +20,9 @@ export const DisplayedAssets: ServiceNameToAssetTypes = { kibana: Object.values(KibanaAssetType), elasticsearch: Object.values(ElasticsearchAssetType), }; +export type DisplayedAssetType = KibanaAssetType | ElasticsearchAssetType; -export const AssetTitleMap: Record = { +export const AssetTitleMap: Record = { dashboard: 'Dashboard', ilm_policy: 'ILM Policy', ingest_pipeline: 'Ingest Pipeline', @@ -31,7 +32,6 @@ export const AssetTitleMap: Record = { component_template: 'Component Template', search: 'Saved Search', visualization: 'Visualization', - input: 'Agent input', map: 'Map', data_stream_ilm_policy: 'Data Stream ILM Policy', lens: 'Lens', diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx index 6ddff968bd3f3b..41db09b0538b91 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/constants.tsx @@ -7,7 +7,7 @@ import type { IconType } from '@elastic/eui'; -import type { AssetType, ServiceName } from '../../types'; +import type { ServiceName } from '../../types'; import { ElasticsearchAssetType, KibanaAssetType } from '../../types'; // only allow Kibana assets for the kibana key, ES asssets for elasticsearch, etc @@ -19,7 +19,9 @@ export const DisplayedAssets: ServiceNameToAssetTypes = { elasticsearch: Object.values(ElasticsearchAssetType), }; -export const AssetTitleMap: Record = { +export type DisplayedAssetType = ElasticsearchAssetType | KibanaAssetType; + +export const AssetTitleMap: Record = { dashboard: 'Dashboard', ilm_policy: 'ILM Policy', ingest_pipeline: 'Ingest Pipeline', @@ -29,7 +31,6 @@ export const AssetTitleMap: Record = { component_template: 'Component Template', search: 'Saved Search', visualization: 'Visualization', - input: 'Agent input', map: 'Map', data_stream_ilm_policy: 'Data Stream ILM Policy', lens: 'Lens', diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx index 487df179803452..0a601d2128bbae 100644 --- a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/details.tsx @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -import React, { memo, useMemo } from 'react'; +import React, { memo, useCallback, useMemo, useState } from 'react'; import { FormattedMessage } from '@kbn/i18n/react'; import { EuiFlexGroup, @@ -13,6 +13,8 @@ import { EuiTextColor, EuiDescriptionList, EuiNotificationBadge, + EuiLink, + EuiPortal, } from '@elastic/eui'; import type { EuiDescriptionListProps } from '@elastic/eui/src/components/description_list/description_list'; @@ -26,6 +28,8 @@ import { entries } from '../../../../../types'; import { useGetCategories } from '../../../../../hooks'; import { AssetTitleMap, DisplayedAssets, ServiceTitleMap } from '../../../constants'; +import { NoticeModal } from './notice_modal'; + interface Props { packageInfo: PackageInfo; } @@ -41,6 +45,11 @@ export const Details: React.FC = memo(({ packageInfo }) => { return []; }, [categoriesData, isLoadingCategories, packageInfo.categories]); + const [isNoticeModalOpen, setIsNoticeModalOpen] = useState(false); + const toggleNoticeModal = useCallback(() => { + setIsNoticeModalOpen(!isNoticeModalOpen); + }, [isNoticeModalOpen]); + const listItems = useMemo(() => { // Base details: version and categories const items: EuiDescriptionListProps['listItems'] = [ @@ -123,14 +132,23 @@ export const Details: React.FC = memo(({ packageInfo }) => { } // License details - if (packageInfo.license) { + if (packageInfo.license || packageInfo.notice) { items.push({ title: ( ), - description: packageInfo.license, + description: ( + <> +

{packageInfo.license}

+ {packageInfo.notice && ( +

+ NOTICE.txt +

+ )} + + ), }); } @@ -140,21 +158,30 @@ export const Details: React.FC = memo(({ packageInfo }) => { packageInfo.assets, packageInfo.data_streams, packageInfo.license, + packageInfo.notice, packageInfo.version, + toggleNoticeModal, ]); return ( - - - -

- -

- - - - - - + <> + + {isNoticeModalOpen && packageInfo.notice && ( + + )} + + + + +

+ +

+ + + + + + + ); }); diff --git a/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/notice_modal.tsx b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/notice_modal.tsx new file mode 100644 index 00000000000000..239bd133d3c919 --- /dev/null +++ b/x-pack/plugins/fleet/public/applications/integrations/sections/epm/screens/detail/overview/notice_modal.tsx @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React, { useEffect, useState } from 'react'; +import { + EuiCodeBlock, + EuiLoadingContent, + EuiModal, + EuiModalBody, + EuiModalHeader, + EuiModalFooter, + EuiModalHeaderTitle, + EuiButton, +} from '@elastic/eui'; +import { i18n } from '@kbn/i18n'; +import { FormattedMessage } from '@kbn/i18n/react'; + +import { sendGetFileByPath, useStartServices } from '../../../../../hooks'; + +interface Props { + noticePath: string; + onClose: () => void; +} + +export const NoticeModal: React.FunctionComponent = ({ noticePath, onClose }) => { + const { notifications } = useStartServices(); + const [notice, setNotice] = useState(undefined); + + useEffect(() => { + async function fetchData() { + try { + const { data } = await sendGetFileByPath(noticePath); + setNotice(data || ''); + } catch (err) { + notifications.toasts.addError(err, { + title: i18n.translate('xpack.fleet.epm.errorLoadingNotice', { + defaultMessage: 'Error loading NOTICE.txt', + }), + }); + } + } + fetchData(); + }, [noticePath, notifications]); + return ( + + + +

NOTICE.txt

+ + + + + {notice ? ( + notice + ) : ( + // Simulate a long notice while loading + <> +

+ +

+

+ +

+ + )} + + + + + + + + + ); +}; diff --git a/x-pack/plugins/fleet/server/services/epm/archive/index.ts b/x-pack/plugins/fleet/server/services/epm/archive/index.ts index 809684df0592c9..b08ec815a394d0 100644 --- a/x-pack/plugins/fleet/server/services/epm/archive/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/archive/index.ts @@ -114,6 +114,13 @@ export function getPathParts(path: string): AssetParts { [pkgkey, service, type, file] = path.replace(`data_stream/${dataset}/`, '').split('/'); } + // To support the NOTICE asset at the root level + if (service === 'NOTICE.txt') { + file = service; + type = 'notice'; + service = ''; + } + // This is to cover for the fields.yml files inside the "fields" directory if (file === undefined) { file = type; diff --git a/x-pack/plugins/fleet/server/services/epm/packages/get.ts b/x-pack/plugins/fleet/server/services/epm/packages/get.ts index e4e4f9c55fd2be..404431816d10c5 100644 --- a/x-pack/plugins/fleet/server/services/epm/packages/get.ts +++ b/x-pack/plugins/fleet/server/services/epm/packages/get.ts @@ -126,6 +126,7 @@ export async function getPackageInfo(options: { title: packageInfo.title || nameAsTitle(packageInfo.name), assets: Registry.groupPathsByService(paths || []), removable: !isRequiredPackage(pkgName), + notice: Registry.getNoticePath(paths || []), }; const updated = { ...packageInfo, ...additions }; diff --git a/x-pack/plugins/fleet/server/services/epm/registry/index.ts b/x-pack/plugins/fleet/server/services/epm/registry/index.ts index 5ee7e198555c59..011a0e74e8c184 100644 --- a/x-pack/plugins/fleet/server/services/epm/registry/index.ts +++ b/x-pack/plugins/fleet/server/services/epm/registry/index.ts @@ -255,3 +255,15 @@ export function groupPathsByService(paths: string[]): AssetsGroupedByServiceByTy elasticsearch: assets.elasticsearch, }; } + +export function getNoticePath(paths: string[]): string | undefined { + for (const path of paths) { + const parts = getPathParts(path.replace(/^\/package\//, '')); + if (parts.type === 'notice') { + const { pkgName, pkgVersion } = splitPkgKey(parts.pkgkey); + return `/package/${pkgName}/${pkgVersion}/${parts.file}`; + } + } + + return undefined; +} From 571524005ee9913f50eaac73a8689c0e33439e3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Casper=20H=C3=BCbertz?= Date: Mon, 14 Jun 2021 18:34:52 +0200 Subject: [PATCH 07/77] [APM] Change impact indicator size (#102060) --- .../shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap | 2 +- x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/apm/public/components/shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap b/x-pack/plugins/apm/public/components/shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap index 8e89939f585aae..87b5b68e260267 100644 --- a/x-pack/plugins/apm/public/components/shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap +++ b/x-pack/plugins/apm/public/components/shared/ImpactBar/__snapshots__/ImpactBar.test.js.snap @@ -4,7 +4,7 @@ exports[`ImpactBar component should render with default values 1`] = ` `; diff --git a/x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx b/x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx index 92f488b8ba0ee3..87b3c669e993c5 100644 --- a/x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx +++ b/x-pack/plugins/apm/public/components/shared/ImpactBar/index.tsx @@ -18,7 +18,7 @@ export interface ImpactBarProps extends Record { export function ImpactBar({ value, - size = 'l', + size = 'm', max = 100, color = 'primary', ...rest From 666bce392383a57f983617bc25b901a556ec20b0 Mon Sep 17 00:00:00 2001 From: Dario Gieselaar Date: Mon, 14 Jun 2021 18:44:29 +0200 Subject: [PATCH 08/77] [APM] Enforce span creation/naming for ES searches (#101856) --- packages/kbn-apm-utils/src/index.ts | 51 ++- .../chart_preview/get_transaction_duration.ts | 120 +++--- .../get_transaction_error_count.ts | 72 ++-- .../get_transaction_error_rate.ts | 5 +- ...et_correlations_for_failed_transactions.ts | 207 +++++----- .../errors/get_overall_error_timeseries.ts | 58 +-- .../get_correlations_for_slow_transactions.ts | 89 ++--- .../latency/get_duration_for_percentile.ts | 41 +- .../latency/get_latency_distribution.ts | 116 +++--- .../correlations/latency/get_max_latency.ts | 61 ++- .../get_overall_latency_distribution.ts | 5 +- .../lib/environments/get_all_environments.ts | 88 +++-- .../lib/environments/get_environments.ts | 83 ++-- .../__snapshots__/get_buckets.test.ts.snap | 1 + .../errors/distribution/get_buckets.test.ts | 2 +- .../lib/errors/distribution/get_buckets.ts | 90 ++--- .../lib/errors/get_error_group_sample.ts | 77 ++-- .../apm/server/lib/errors/get_error_groups.ts | 139 ++++--- .../helpers/aggregated_transactions/index.ts | 18 +- .../create_es_client/call_async_with_debug.ts | 21 +- .../create_apm_event_client/index.test.ts | 2 +- .../create_apm_event_client/index.ts | 19 +- .../create_internal_es_client/index.ts | 44 ++- .../server/lib/helpers/setup_request.test.ts | 32 +- .../java/gc/fetch_and_transform_gc_metrics.ts | 4 +- .../by_agent/java/gc/get_gc_rate_chart.ts | 22 +- .../by_agent/java/gc/get_gc_time_chart.ts | 22 +- .../by_agent/java/heap_memory/index.ts | 34 +- .../by_agent/java/non_heap_memory/index.ts | 38 +- .../by_agent/java/thread_count/index.ts | 30 +- .../lib/metrics/by_agent/shared/cpu/index.ts | 32 +- .../metrics/by_agent/shared/memory/index.ts | 70 ++-- .../metrics/fetch_and_transform_metrics.ts | 4 +- .../get_service_count.ts | 50 +-- .../get_transactions_per_minute.ts | 113 +++--- .../lib/observability_overview/has_data.ts | 48 +-- .../lib/rum_client/get_client_metrics.ts | 2 +- .../server/lib/rum_client/get_js_errors.ts | 2 +- .../lib/rum_client/get_long_task_metrics.ts | 2 +- .../rum_client/get_page_load_distribution.ts | 7 +- .../lib/rum_client/get_page_view_trends.ts | 2 +- .../lib/rum_client/get_pl_dist_breakdown.ts | 5 +- .../server/lib/rum_client/get_rum_services.ts | 2 +- .../server/lib/rum_client/get_url_search.ts | 2 +- .../lib/rum_client/get_visitor_breakdown.ts | 2 +- .../lib/rum_client/get_web_core_vitals.ts | 2 +- .../apm/server/lib/rum_client/has_rum_data.ts | 2 +- .../ui_filters/local_ui_filters/index.ts | 55 +-- .../fetch_service_paths_from_trace_ids.ts | 120 +++--- .../server/lib/service_map/get_service_map.ts | 101 ++--- .../get_service_map_service_node_info.ts | 119 +++--- .../lib/service_map/get_trace_sample_ids.ts | 195 +++++----- .../apm/server/lib/service_nodes/index.ts | 101 +++-- .../__snapshots__/queries.test.ts.snap | 1 + .../get_derived_service_annotations.ts | 142 ++++--- .../lib/services/get_service_agent_name.ts | 68 ++-- .../get_destination_map.ts | 137 ++++--- .../get_service_dependencies/get_metrics.ts | 94 ++--- ...service_error_group_detailed_statistics.ts | 111 +++--- ...get_service_error_group_main_statistics.ts | 45 ++- .../get_service_error_groups/index.ts | 76 ++-- .../get_service_instance_metadata_details.ts | 56 +-- ...vice_instances_system_metric_statistics.ts | 259 +++++++------ ...ervice_instances_transaction_statistics.ts | 221 ++++++----- .../services/get_service_metadata_details.ts | 172 ++++----- .../services/get_service_metadata_icons.ts | 92 ++--- .../lib/services/get_service_node_metadata.ts | 69 ++-- ...e_transaction_group_detailed_statistics.ts | 191 +++++----- .../get_service_transaction_groups.ts | 86 ++--- .../services/get_service_transaction_types.ts | 68 ++-- .../get_services/get_legacy_data_status.ts | 44 +-- .../get_service_transaction_stats.ts | 142 +++---- .../get_services_from_metric_documents.ts | 36 +- .../get_services/has_historical_agent_data.ts | 35 +- .../apm/server/lib/services/get_throughput.ts | 25 +- .../get_service_profiling_statistics.ts | 125 +++---- .../get_service_profiling_timeline.ts | 50 +-- .../apm/server/lib/services/queries.test.ts | 2 +- .../create_or_update_configuration.ts | 43 +-- .../delete_configuration.ts | 17 +- .../find_exact_configuration.ts | 56 ++- .../get_agent_name_by_service.ts | 54 +-- .../get_existing_environments_for_service.ts | 54 +-- .../agent_configuration/get_service_names.ts | 66 ++-- .../list_configurations.ts | 6 +- .../mark_applied_by_agent.ts | 5 +- .../search_configurations.ts | 98 +++-- .../create_or_update_custom_link.test.ts | 52 +-- .../create_or_update_custom_link.ts | 31 +- .../custom_link/delete_custom_link.ts | 17 +- .../settings/custom_link/get_transaction.ts | 56 +-- .../settings/custom_link/list_custom_links.ts | 78 ++-- .../apm/server/lib/traces/get_trace_items.ts | 166 ++++---- .../lib/transaction_groups/get_error_rate.ts | 132 +++---- .../get_transaction_group_stats.ts | 187 +++++----- .../lib/transaction_groups/queries.test.ts | 4 +- .../lib/transactions/breakdown/index.ts | 353 +++++++++--------- .../distribution/get_buckets/index.ts | 128 ++++--- .../distribution/get_distribution_max.ts | 70 ++-- .../transactions/get_latency_charts/index.ts | 69 ++-- .../get_throughput_charts/index.ts | 41 +- .../lib/transactions/get_transaction/index.ts | 39 +- .../get_transaction_by_trace/index.ts | 59 +-- .../plugins/apm/server/utils/test_helpers.tsx | 2 +- .../plugins/apm/server/utils/with_apm_span.ts | 2 +- 105 files changed, 3410 insertions(+), 3451 deletions(-) diff --git a/packages/kbn-apm-utils/src/index.ts b/packages/kbn-apm-utils/src/index.ts index 384b6683199e5b..09a6989091f609 100644 --- a/packages/kbn-apm-utils/src/index.ts +++ b/packages/kbn-apm-utils/src/index.ts @@ -14,6 +14,7 @@ export interface SpanOptions { type?: string; subtype?: string; labels?: Record; + intercept?: boolean; } type Span = Exclude; @@ -36,23 +37,27 @@ export async function withSpan( ): Promise { const options = parseSpanOptions(optionsOrName); - const { name, type, subtype, labels } = options; + const { name, type, subtype, labels, intercept } = options; if (!agent.isStarted()) { return cb(); } + let createdSpan: Span | undefined; + // When a span starts, it's marked as the active span in its context. // When it ends, it's not untracked, which means that if a span // starts directly after this one ends, the newly started span is a // child of this span, even though it should be a sibling. // To mitigate this, we queue a microtask by awaiting a promise. - await Promise.resolve(); + if (!intercept) { + await Promise.resolve(); - const span = agent.startSpan(name); + createdSpan = agent.startSpan(name) ?? undefined; - if (!span) { - return cb(); + if (!createdSpan) { + return cb(); + } } // If a span is created in the same context as the span that we just @@ -61,33 +66,51 @@ export async function withSpan( // mitigate this we create a new context. return runInNewContext(() => { + const promise = cb(createdSpan); + + let span: Span | undefined = createdSpan; + + if (intercept) { + span = agent.currentSpan ?? undefined; + } + + if (!span) { + return promise; + } + + const targetedSpan = span; + + if (name) { + targetedSpan.name = name; + } + // @ts-ignore if (type) { - span.type = type; + targetedSpan.type = type; } if (subtype) { - span.subtype = subtype; + targetedSpan.subtype = subtype; } if (labels) { - span.addLabels(labels); + targetedSpan.addLabels(labels); } - return cb(span) + return promise .then((res) => { - if (!span.outcome || span.outcome === 'unknown') { - span.outcome = 'success'; + if (!targetedSpan.outcome || targetedSpan.outcome === 'unknown') { + targetedSpan.outcome = 'success'; } return res; }) .catch((err) => { - if (!span.outcome || span.outcome === 'unknown') { - span.outcome = 'failure'; + if (!targetedSpan.outcome || targetedSpan.outcome === 'unknown') { + targetedSpan.outcome = 'failure'; } throw err; }) .finally(() => { - span.end(); + targetedSpan.end(); }); }); } diff --git a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_duration.ts b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_duration.ts index 091982598d6a3e..6ce175fcb83626 100644 --- a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_duration.ts +++ b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_duration.ts @@ -15,82 +15,82 @@ import { import { ProcessorEvent } from '../../../../common/processor_event'; import { environmentQuery, rangeQuery } from '../../../../server/utils/queries'; import { AlertParams } from '../../../routes/alerts/chart_preview'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -export function getTransactionDurationChartPreview({ +export async function getTransactionDurationChartPreview({ alertParams, setup, }: { alertParams: AlertParams; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_transaction_duration_chart_preview', async () => { - const { apmEventClient, start, end } = setup; - const { - aggregationType, - environment, - serviceName, - transactionType, - } = alertParams; + const { apmEventClient, start, end } = setup; + const { + aggregationType, + environment, + serviceName, + transactionType, + } = alertParams; - const query = { - bool: { - filter: [ - { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, - ...(serviceName ? [{ term: { [SERVICE_NAME]: serviceName } }] : []), - ...(transactionType - ? [{ term: { [TRANSACTION_TYPE]: transactionType } }] - : []), - ...rangeQuery(start, end), - ...environmentQuery(environment), - ] as QueryDslQueryContainer[], - }, - }; + const query = { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: ProcessorEvent.transaction } }, + ...(serviceName ? [{ term: { [SERVICE_NAME]: serviceName } }] : []), + ...(transactionType + ? [{ term: { [TRANSACTION_TYPE]: transactionType } }] + : []), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ] as QueryDslQueryContainer[], + }, + }; - const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); + const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); - const aggs = { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - }, - aggs: { - agg: - aggregationType === 'avg' - ? { avg: { field: TRANSACTION_DURATION } } - : { - percentiles: { - field: TRANSACTION_DURATION, - percents: [aggregationType === '95th' ? 95 : 99], - }, + const aggs = { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + }, + aggs: { + agg: + aggregationType === 'avg' + ? { avg: { field: TRANSACTION_DURATION } } + : { + percentiles: { + field: TRANSACTION_DURATION, + percents: [aggregationType === '95th' ? 95 : 99], }, - }, + }, }, - }; - const params = { - apm: { events: [ProcessorEvent.transaction] }, - body: { size: 0, query, aggs }, - }; - const resp = await apmEventClient.search(params); + }, + }; + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { size: 0, query, aggs }, + }; + const resp = await apmEventClient.search( + 'get_transaction_duration_chart_preview', + params + ); - if (!resp.aggregations) { - return []; - } + if (!resp.aggregations) { + return []; + } - return resp.aggregations.timeseries.buckets.map((bucket) => { - const percentilesKey = aggregationType === '95th' ? '95.0' : '99.0'; - const x = bucket.key; - const y = - aggregationType === 'avg' - ? (bucket.agg as { value: number | null }).value - : (bucket.agg as { values: Record }).values[ - percentilesKey - ]; + return resp.aggregations.timeseries.buckets.map((bucket) => { + const percentilesKey = aggregationType === '95th' ? '95.0' : '99.0'; + const x = bucket.key; + const y = + aggregationType === 'avg' + ? (bucket.agg as { value: number | null }).value + : (bucket.agg as { values: Record }).values[ + percentilesKey + ]; - return { x, y }; - }); + return { x, y }; }); } diff --git a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_count.ts b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_count.ts index 2cf1317dc44b0d..3d64c63cb2041c 100644 --- a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_count.ts +++ b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_count.ts @@ -9,58 +9,58 @@ import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; import { AlertParams } from '../../../routes/alerts/chart_preview'; import { environmentQuery, rangeQuery } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -export function getTransactionErrorCountChartPreview({ +export async function getTransactionErrorCountChartPreview({ setup, alertParams, }: { setup: Setup & SetupTimeRange; alertParams: AlertParams; }) { - return withApmSpan('get_transaction_error_count_chart_preview', async () => { - const { apmEventClient, start, end } = setup; - const { serviceName, environment } = alertParams; + const { apmEventClient, start, end } = setup; + const { serviceName, environment } = alertParams; - const query = { - bool: { - filter: [ - ...(serviceName ? [{ term: { [SERVICE_NAME]: serviceName } }] : []), - ...rangeQuery(start, end), - ...environmentQuery(environment), - ], - }, - }; + const query = { + bool: { + filter: [ + ...(serviceName ? [{ term: { [SERVICE_NAME]: serviceName } }] : []), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ], + }, + }; - const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); + const { intervalString } = getBucketSize({ start, end, numBuckets: 20 }); - const aggs = { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - }, + const aggs = { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, }, - }; + }, + }; - const params = { - apm: { events: [ProcessorEvent.error] }, - body: { size: 0, query, aggs }, - }; + const params = { + apm: { events: [ProcessorEvent.error] }, + body: { size: 0, query, aggs }, + }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search( + 'get_transaction_error_count_chart_preview', + params + ); - if (!resp.aggregations) { - return []; - } + if (!resp.aggregations) { + return []; + } - return resp.aggregations.timeseries.buckets.map((bucket) => { - return { - x: bucket.key, - y: bucket.doc_count, - }; - }); + return resp.aggregations.timeseries.buckets.map((bucket) => { + return { + x: bucket.key, + y: bucket.doc_count, + }; }); } diff --git a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts index f0c8d23e0e8fa4..0a6a25ad9c5332 100644 --- a/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/alerts/chart_preview/get_transaction_error_rate.ts @@ -64,7 +64,10 @@ export async function getTransactionErrorRateChartPreview({ body: { size: 0, query, aggs }, }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search( + 'get_transaction_error_rate_chart_preview', + params + ); if (!resp.aggregations) { return []; diff --git a/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts b/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts index 8ee469c9a93c77..11e9f99ddb356b 100644 --- a/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts +++ b/x-pack/plugins/apm/server/lib/correlations/errors/get_correlations_for_failed_transactions.ts @@ -21,68 +21,68 @@ import { getTimeseriesAggregation, getTransactionErrorRateTimeSeries, } from '../../helpers/transaction_error_rate'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { CorrelationsOptions, getCorrelationsFilters } from '../get_filters'; interface Options extends CorrelationsOptions { fieldNames: string[]; } export async function getCorrelationsForFailedTransactions(options: Options) { - return withApmSpan('get_correlations_for_failed_transactions', async () => { - const { fieldNames, setup } = options; - const { apmEventClient } = setup; - const filters = getCorrelationsFilters(options); - - const params = { - apm: { events: [ProcessorEvent.transaction] }, - track_total_hits: true, - body: { - size: 0, - query: { - bool: { filter: filters }, - }, - aggs: { - failed_transactions: { - filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, - - // significant term aggs - aggs: fieldNames.reduce((acc, fieldName) => { - return { - ...acc, - [fieldName]: { - significant_terms: { - size: 10, - field: fieldName, - background_filter: { - bool: { - filter: filters, - must_not: { - term: { [EVENT_OUTCOME]: EventOutcome.failure }, - }, + const { fieldNames, setup } = options; + const { apmEventClient } = setup; + const filters = getCorrelationsFilters(options); + + const params = { + apm: { events: [ProcessorEvent.transaction] }, + track_total_hits: true, + body: { + size: 0, + query: { + bool: { filter: filters }, + }, + aggs: { + failed_transactions: { + filter: { term: { [EVENT_OUTCOME]: EventOutcome.failure } }, + + // significant term aggs + aggs: fieldNames.reduce((acc, fieldName) => { + return { + ...acc, + [fieldName]: { + significant_terms: { + size: 10, + field: fieldName, + background_filter: { + bool: { + filter: filters, + must_not: { + term: { [EVENT_OUTCOME]: EventOutcome.failure }, }, }, }, }, - }; - }, {} as Record), - }, + }, + }; + }, {} as Record), }, }, - }; - - const response = await apmEventClient.search(params); - if (!response.aggregations) { - return { significantTerms: [] }; - } - - const sigTermAggs = omit( - response.aggregations?.failed_transactions, - 'doc_count' - ); - - const topSigTerms = processSignificantTermAggs({ sigTermAggs }); - return getErrorRateTimeSeries({ setup, filters, topSigTerms }); - }); + }, + }; + + const response = await apmEventClient.search( + 'get_correlations_for_failed_transactions', + params + ); + if (!response.aggregations) { + return { significantTerms: [] }; + } + + const sigTermAggs = omit( + response.aggregations?.failed_transactions, + 'doc_count' + ); + + const topSigTerms = processSignificantTermAggs({ sigTermAggs }); + return getErrorRateTimeSeries({ setup, filters, topSigTerms }); } export async function getErrorRateTimeSeries({ @@ -94,58 +94,59 @@ export async function getErrorRateTimeSeries({ filters: ESFilter[]; topSigTerms: TopSigTerm[]; }) { - return withApmSpan('get_error_rate_timeseries', async () => { - const { start, end, apmEventClient } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); - - if (isEmpty(topSigTerms)) { - return { significantTerms: [] }; + const { start, end, apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); + + if (isEmpty(topSigTerms)) { + return { significantTerms: [] }; + } + + const timeseriesAgg = getTimeseriesAggregation(start, end, intervalString); + + const perTermAggs = topSigTerms.reduce( + (acc, term, index) => { + acc[`term_${index}`] = { + filter: { term: { [term.fieldName]: term.fieldValue } }, + aggs: { timeseries: timeseriesAgg }, + }; + return acc; + }, + {} as { + [key: string]: { + filter: AggregationOptionsByType['filter']; + aggs: { timeseries: typeof timeseriesAgg }; + }; } - - const timeseriesAgg = getTimeseriesAggregation(start, end, intervalString); - - const perTermAggs = topSigTerms.reduce( - (acc, term, index) => { - acc[`term_${index}`] = { - filter: { term: { [term.fieldName]: term.fieldValue } }, - aggs: { timeseries: timeseriesAgg }, - }; - return acc; - }, - {} as { - [key: string]: { - filter: AggregationOptionsByType['filter']; - aggs: { timeseries: typeof timeseriesAgg }; - }; - } - ); - - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { bool: { filter: filters } }, - aggs: perTermAggs, - }, - }; - - const response = await apmEventClient.search(params); - const { aggregations } = response; - - if (!aggregations) { - return { significantTerms: [] }; - } - - return { - significantTerms: topSigTerms.map((topSig, index) => { - const agg = aggregations[`term_${index}`]!; - - return { - ...topSig, - timeseries: getTransactionErrorRateTimeSeries(agg.timeseries.buckets), - }; - }), - }; - }); + ); + + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: perTermAggs, + }, + }; + + const response = await apmEventClient.search( + 'get_error_rate_timeseries', + params + ); + const { aggregations } = response; + + if (!aggregations) { + return { significantTerms: [] }; + } + + return { + significantTerms: topSigTerms.map((topSig, index) => { + const agg = aggregations[`term_${index}`]!; + + return { + ...topSig, + timeseries: getTransactionErrorRateTimeSeries(agg.timeseries.buckets), + }; + }), + }; } diff --git a/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts b/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts index 9387e64a51e01c..f3477273806b6f 100644 --- a/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts +++ b/x-pack/plugins/apm/server/lib/correlations/errors/get_overall_error_timeseries.ts @@ -11,41 +11,41 @@ import { getTimeseriesAggregation, getTransactionErrorRateTimeSeries, } from '../../helpers/transaction_error_rate'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { CorrelationsOptions, getCorrelationsFilters } from '../get_filters'; export async function getOverallErrorTimeseries(options: CorrelationsOptions) { - return withApmSpan('get_error_rate_timeseries', async () => { - const { setup } = options; - const filters = getCorrelationsFilters(options); - const { start, end, apmEventClient } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); + const { setup } = options; + const filters = getCorrelationsFilters(options); + const { start, end, apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets: 15 }); - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { bool: { filter: filters } }, - aggs: { - timeseries: getTimeseriesAggregation(start, end, intervalString), - }, + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: { + timeseries: getTimeseriesAggregation(start, end, intervalString), }, - }; + }, + }; - const response = await apmEventClient.search(params); - const { aggregations } = response; + const response = await apmEventClient.search( + 'get_error_rate_timeseries', + params + ); + const { aggregations } = response; - if (!aggregations) { - return { overall: null }; - } + if (!aggregations) { + return { overall: null }; + } - return { - overall: { - timeseries: getTransactionErrorRateTimeSeries( - aggregations.timeseries.buckets - ), - }, - }; - }); + return { + overall: { + timeseries: getTransactionErrorRateTimeSeries( + aggregations.timeseries.buckets + ), + }, + }; } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts index 0f93d1411a001c..c37b3e3ab82426 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_correlations_for_slow_transactions.ts @@ -41,60 +41,63 @@ export async function getCorrelationsForSlowTransactions(options: Options) { return { significantTerms: [] }; } - const response = await withApmSpan('get_significant_terms', () => { - const params = { - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { - bool: { - // foreground filters - filter: filters, - must: { - function_score: { - query: { - range: { - [TRANSACTION_DURATION]: { gte: durationForPercentile }, - }, + const params = { + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { + // foreground filters + filter: filters, + must: { + function_score: { + query: { + range: { + [TRANSACTION_DURATION]: { gte: durationForPercentile }, }, - script_score: { - script: { - source: `Math.log(2 + doc['${TRANSACTION_DURATION}'].value)`, - }, + }, + script_score: { + script: { + source: `Math.log(2 + doc['${TRANSACTION_DURATION}'].value)`, }, }, }, }, }, - aggs: fieldNames.reduce((acc, fieldName) => { - return { - ...acc, - [fieldName]: { - significant_terms: { - size: 10, - field: fieldName, - background_filter: { - bool: { - filter: [ - ...filters, - { - range: { - [TRANSACTION_DURATION]: { - lt: durationForPercentile, - }, + }, + aggs: fieldNames.reduce((acc, fieldName) => { + return { + ...acc, + [fieldName]: { + significant_terms: { + size: 10, + field: fieldName, + background_filter: { + bool: { + filter: [ + ...filters, + { + range: { + [TRANSACTION_DURATION]: { + lt: durationForPercentile, }, }, - ], - }, + }, + ], }, }, }, - }; - }, {} as Record), - }, - }; - return apmEventClient.search(params); - }); + }, + }; + }, {} as Record), + }, + }; + + const response = await apmEventClient.search( + 'get_significant_terms', + params + ); + if (!response.aggregations) { return { significantTerms: [] }; } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts index 43c261743861d8..a686980700d83a 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_duration_for_percentile.ts @@ -8,7 +8,6 @@ import { ESFilter } from '../../../../../../../typings/elasticsearch'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; export async function getDurationForPercentile({ @@ -20,31 +19,27 @@ export async function getDurationForPercentile({ filters: ESFilter[]; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_duration_for_percentiles', async () => { - const { apmEventClient } = setup; - const res = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.transaction], + const { apmEventClient } = setup; + const res = await apmEventClient.search('get_duration_for_percentiles', { + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + size: 0, + query: { + bool: { filter: filters }, }, - body: { - size: 0, - query: { - bool: { filter: filters }, - }, - aggs: { - percentile: { - percentiles: { - field: TRANSACTION_DURATION, - percents: [durationPercentile], - }, + aggs: { + percentile: { + percentiles: { + field: TRANSACTION_DURATION, + percents: [durationPercentile], }, }, }, - }); - - const duration = Object.values( - res.aggregations?.percentile.values || {} - )[0]; - return duration || 0; + }, }); + + const duration = Object.values(res.aggregations?.percentile.values || {})[0]; + return duration || 0; } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts index 6d42b26b22e42d..be1bb631378cff 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_latency_distribution.ts @@ -10,7 +10,7 @@ import { ESFilter } from '../../../../../../../typings/elasticsearch'; import { ProcessorEvent } from '../../../../common/processor_event'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { TopSigTerm } from '../process_significant_term_aggs'; -import { withApmSpan } from '../../../utils/with_apm_span'; + import { getDistributionAggregation, trimBuckets, @@ -29,70 +29,70 @@ export async function getLatencyDistribution({ maxLatency: number; distributionInterval: number; }) { - return withApmSpan('get_latency_distribution', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const distributionAgg = getDistributionAggregation( - maxLatency, - distributionInterval - ); + const distributionAgg = getDistributionAggregation( + maxLatency, + distributionInterval + ); - const perTermAggs = topSigTerms.reduce( - (acc, term, index) => { - acc[`term_${index}`] = { - filter: { term: { [term.fieldName]: term.fieldValue } }, - aggs: { - distribution: distributionAgg, - }, + const perTermAggs = topSigTerms.reduce( + (acc, term, index) => { + acc[`term_${index}`] = { + filter: { term: { [term.fieldName]: term.fieldValue } }, + aggs: { + distribution: distributionAgg, + }, + }; + return acc; + }, + {} as Record< + string, + { + filter: AggregationOptionsByType['filter']; + aggs: { + distribution: typeof distributionAgg; }; - return acc; - }, - {} as Record< - string, - { - filter: AggregationOptionsByType['filter']; - aggs: { - distribution: typeof distributionAgg; - }; - } - > - ); + } + > + ); - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { bool: { filter: filters } }, - aggs: perTermAggs, - }, - }; + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { bool: { filter: filters } }, + aggs: perTermAggs, + }, + }; - const response = await withApmSpan('get_terms_distribution', () => - apmEventClient.search(params) - ); - type Agg = NonNullable; + const response = await apmEventClient.search( + 'get_latency_distribution', + params + ); - if (!response.aggregations) { - return []; - } + type Agg = NonNullable; - return topSigTerms.map((topSig, index) => { - // ignore the typescript error since existence of response.aggregations is already checked: - // @ts-expect-error - const agg = response.aggregations[`term_${index}`] as Agg[string]; - const total = agg.distribution.doc_count; - const buckets = trimBuckets( - agg.distribution.dist_filtered_by_latency.buckets - ); + if (!response.aggregations) { + return []; + } - return { - ...topSig, - distribution: buckets.map((bucket) => ({ - x: bucket.key, - y: (bucket.doc_count / total) * 100, - })), - }; - }); + return topSigTerms.map((topSig, index) => { + // ignore the typescript error since existence of response.aggregations is already checked: + // @ts-expect-error + const agg = response.aggregations[`term_${index}`] as Agg[string]; + const total = agg.distribution.doc_count; + const buckets = trimBuckets( + agg.distribution.dist_filtered_by_latency.buckets + ); + + return { + ...topSig, + distribution: buckets.map((bucket) => ({ + x: bucket.key, + y: (bucket.doc_count / total) * 100, + })), + }; }); } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts index 8b415bf0d80a7b..f2762086614b49 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_max_latency.ts @@ -8,7 +8,6 @@ import { ESFilter } from '../../../../../../../typings/elasticsearch'; import { TRANSACTION_DURATION } from '../../../../common/elasticsearch_fieldnames'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { TopSigTerm } from '../process_significant_term_aggs'; @@ -21,41 +20,39 @@ export async function getMaxLatency({ filters: ESFilter[]; topSigTerms?: TopSigTerm[]; }) { - return withApmSpan('get_max_latency', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const params = { - // TODO: add support for metrics - apm: { events: [ProcessorEvent.transaction] }, - body: { - size: 0, - query: { - bool: { - filter: filters, + const params = { + // TODO: add support for metrics + apm: { events: [ProcessorEvent.transaction] }, + body: { + size: 0, + query: { + bool: { + filter: filters, - ...(topSigTerms.length - ? { - // only include docs containing the significant terms - should: topSigTerms.map((term) => ({ - term: { [term.fieldName]: term.fieldValue }, - })), - minimum_should_match: 1, - } - : null), - }, + ...(topSigTerms.length + ? { + // only include docs containing the significant terms + should: topSigTerms.map((term) => ({ + term: { [term.fieldName]: term.fieldValue }, + })), + minimum_should_match: 1, + } + : null), }, - aggs: { - // TODO: add support for metrics - // max_latency: { max: { field: TRANSACTION_DURATION } }, - max_latency: { - percentiles: { field: TRANSACTION_DURATION, percents: [99] }, - }, + }, + aggs: { + // TODO: add support for metrics + // max_latency: { max: { field: TRANSACTION_DURATION } }, + max_latency: { + percentiles: { field: TRANSACTION_DURATION, percents: [99] }, }, }, - }; + }, + }; - const response = await apmEventClient.search(params); - // return response.aggregations?.max_latency.value; - return Object.values(response.aggregations?.max_latency.values ?? {})[0]; - }); + const response = await apmEventClient.search('get_max_latency', params); + // return response.aggregations?.max_latency.value; + return Object.values(response.aggregations?.max_latency.values ?? {})[0]; } diff --git a/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts b/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts index c5d4def51ea54a..b0e0f22c703663 100644 --- a/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts +++ b/x-pack/plugins/apm/server/lib/correlations/latency/get_overall_latency_distribution.ts @@ -71,8 +71,9 @@ export async function getOverallLatencyDistribution( }, }; - const response = await withApmSpan('get_terms_distribution', () => - apmEventClient.search(params) + const response = await apmEventClient.search( + 'get_terms_distribution', + params ); if (!response.aggregations) { diff --git a/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts b/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts index 1bf01c24776fb0..f6a19879748532 100644 --- a/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts +++ b/x-pack/plugins/apm/server/lib/environments/get_all_environments.ts @@ -13,7 +13,6 @@ import { } from '../../../common/elasticsearch_fieldnames'; import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; /** * This is used for getting *all* environments, and does not filter by range. @@ -30,59 +29,56 @@ export async function getAllEnvironments({ searchAggregatedTransactions: boolean; includeMissing?: boolean; }) { - const spanName = serviceName + const operationName = serviceName ? 'get_all_environments_for_service' : 'get_all_environments_for_all_services'; - return withApmSpan(spanName, async () => { - const { apmEventClient, config } = setup; - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; - // omit filter for service.name if "All" option is selected - const serviceNameFilter = serviceName - ? [{ term: { [SERVICE_NAME]: serviceName } }] - : []; + const { apmEventClient, config } = setup; + const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - // use timeout + min_doc_count to return as early as possible - // if filter is not defined to prevent timeouts - ...(!serviceName ? { timeout: '1ms' } : {}), - size: 0, - query: { - bool: { - filter: [...serviceNameFilter], - }, + // omit filter for service.name if "All" option is selected + const serviceNameFilter = serviceName + ? [{ term: { [SERVICE_NAME]: serviceName } }] + : []; + + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + // use timeout + min_doc_count to return as early as possible + // if filter is not defined to prevent timeouts + ...(!serviceName ? { timeout: '1ms' } : {}), + size: 0, + query: { + bool: { + filter: [...serviceNameFilter], }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - size: maxServiceEnvironments, - ...(!serviceName ? { min_doc_count: 0 } : {}), - missing: includeMissing - ? ENVIRONMENT_NOT_DEFINED.value - : undefined, - }, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + size: maxServiceEnvironments, + ...(!serviceName ? { min_doc_count: 0 } : {}), + missing: includeMissing ? ENVIRONMENT_NOT_DEFINED.value : undefined, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search(operationName, params); - const environments = - resp.aggregations?.environments.buckets.map( - (bucket) => bucket.key as string - ) || []; - return environments; - }); + const environments = + resp.aggregations?.environments.buckets.map( + (bucket) => bucket.key as string + ) || []; + return environments; } diff --git a/x-pack/plugins/apm/server/lib/environments/get_environments.ts b/x-pack/plugins/apm/server/lib/environments/get_environments.ts index 509e4cdcd67ac0..c0b267f180010a 100644 --- a/x-pack/plugins/apm/server/lib/environments/get_environments.ts +++ b/x-pack/plugins/apm/server/lib/environments/get_environments.ts @@ -12,7 +12,6 @@ import { import { ENVIRONMENT_NOT_DEFINED } from '../../../common/environment_filter_values'; import { ProcessorEvent } from '../../../common/processor_event'; import { rangeQuery } from '../../../server/utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; @@ -29,60 +28,58 @@ export async function getEnvironments({ serviceName?: string; searchAggregatedTransactions: boolean; }) { - const spanName = serviceName + const operationName = serviceName ? 'get_environments_for_service' : 'get_environments'; - return withApmSpan(spanName, async () => { - const { start, end, apmEventClient, config } = setup; + const { start, end, apmEventClient, config } = setup; - const filter = rangeQuery(start, end); + const filter = rangeQuery(start, end); - if (serviceName) { - filter.push({ - term: { [SERVICE_NAME]: serviceName }, - }); - } + if (serviceName) { + filter.push({ + term: { [SERVICE_NAME]: serviceName }, + }); + } - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; + const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.metric, - ProcessorEvent.error, - ], - }, - body: { - size: 0, - query: { - bool: { - filter, - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.metric, + ProcessorEvent.error, + ], + }, + body: { + size: 0, + query: { + bool: { + filter, }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - missing: ENVIRONMENT_NOT_DEFINED.value, - size: maxServiceEnvironments, - }, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + missing: ENVIRONMENT_NOT_DEFINED.value, + size: maxServiceEnvironments, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); - const aggs = resp.aggregations; - const environmentsBuckets = aggs?.environments.buckets || []; + const resp = await apmEventClient.search(operationName, params); + const aggs = resp.aggregations; + const environmentsBuckets = aggs?.environments.buckets || []; - const environments = environmentsBuckets.map( - (environmentBucket) => environmentBucket.key as string - ); + const environments = environmentsBuckets.map( + (environmentBucket) => environmentBucket.key as string + ); - return environments; - }); + return environments; } diff --git a/x-pack/plugins/apm/server/lib/errors/distribution/__snapshots__/get_buckets.test.ts.snap b/x-pack/plugins/apm/server/lib/errors/distribution/__snapshots__/get_buckets.test.ts.snap index 43fe4dfe752e65..2c0330f17320de 100644 --- a/x-pack/plugins/apm/server/lib/errors/distribution/__snapshots__/get_buckets.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/errors/distribution/__snapshots__/get_buckets.test.ts.snap @@ -3,6 +3,7 @@ exports[`get buckets should make the correct query 1`] = ` Array [ Array [ + "get_error_distribution_buckets", Object { "apm": Object { "events": Array [ diff --git a/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.test.ts b/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.test.ts index b1260d653f3de8..712343d445d44a 100644 --- a/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.test.ts +++ b/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.test.ts @@ -65,7 +65,7 @@ describe('get buckets', () => { }); it('should limit query results to error documents', () => { - const query = clientSpy.mock.calls[0][0]; + const query = clientSpy.mock.calls[0][1]; expect(query.apm.events).toEqual([ProcessorEvent.error]); }); }); diff --git a/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.ts b/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.ts index 462c9bcdc43101..a51464764f2b44 100644 --- a/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.ts +++ b/x-pack/plugins/apm/server/lib/errors/distribution/get_buckets.ts @@ -16,7 +16,6 @@ import { rangeQuery, kqlQuery, } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; export async function getBuckets({ @@ -34,58 +33,59 @@ export async function getBuckets({ bucketSize: number; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_error_distribution_buckets', async () => { - const { start, end, apmEventClient } = setup; - const filter: ESFilter[] = [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ]; + const { start, end, apmEventClient } = setup; + const filter: ESFilter[] = [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ]; - if (groupId) { - filter.push({ term: { [ERROR_GROUP_ID]: groupId } }); - } + if (groupId) { + filter.push({ term: { [ERROR_GROUP_ID]: groupId } }); + } - const params = { - apm: { - events: [ProcessorEvent.error], - }, - body: { - size: 0, - query: { - bool: { - filter, - }, + const params = { + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: 0, + query: { + bool: { + filter, }, - aggs: { - distribution: { - histogram: { - field: '@timestamp', - min_doc_count: 0, - interval: bucketSize, - extended_bounds: { - min: start, - max: end, - }, + }, + aggs: { + distribution: { + histogram: { + field: '@timestamp', + min_doc_count: 0, + interval: bucketSize, + extended_bounds: { + min: start, + max: end, }, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search( + 'get_error_distribution_buckets', + params + ); - const buckets = (resp.aggregations?.distribution.buckets || []).map( - (bucket) => ({ - key: bucket.key, - count: bucket.doc_count, - }) - ); + const buckets = (resp.aggregations?.distribution.buckets || []).map( + (bucket) => ({ + key: bucket.key, + count: bucket.doc_count, + }) + ); - return { - noHits: resp.hits.total.value === 0, - buckets: resp.hits.total.value > 0 ? buckets : [], - }; - }); + return { + noHits: resp.hits.total.value === 0, + buckets: resp.hits.total.value > 0 ? buckets : [], + }; } diff --git a/x-pack/plugins/apm/server/lib/errors/get_error_group_sample.ts b/x-pack/plugins/apm/server/lib/errors/get_error_group_sample.ts index 57fb4861809932..a915a4fb033051 100644 --- a/x-pack/plugins/apm/server/lib/errors/get_error_group_sample.ts +++ b/x-pack/plugins/apm/server/lib/errors/get_error_group_sample.ts @@ -17,11 +17,10 @@ import { rangeQuery, kqlQuery, } from '../../../server/utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getTransaction } from '../transactions/get_transaction'; -export function getErrorGroupSample({ +export async function getErrorGroupSample({ environment, kuery, serviceName, @@ -34,48 +33,46 @@ export function getErrorGroupSample({ groupId: string; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_error_group_sample', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const params = { - apm: { - events: [ProcessorEvent.error as const], - }, - body: { - size: 1, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [ERROR_GROUP_ID]: groupId } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], - should: [{ term: { [TRANSACTION_SAMPLED]: true } }], - }, + const params = { + apm: { + events: [ProcessorEvent.error as const], + }, + body: { + size: 1, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [ERROR_GROUP_ID]: groupId } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + should: [{ term: { [TRANSACTION_SAMPLED]: true } }], }, - sort: asMutableArray([ - { _score: 'desc' }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top - { '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error - ] as const), }, - }; + sort: asMutableArray([ + { _score: 'desc' }, // sort by _score first to ensure that errors with transaction.sampled:true ends up on top + { '@timestamp': { order: 'desc' } }, // sort by timestamp to get the most recent error + ] as const), + }, + }; - const resp = await apmEventClient.search(params); - const error = resp.hits.hits[0]?._source; - const transactionId = error?.transaction?.id; - const traceId = error?.trace?.id; + const resp = await apmEventClient.search('get_error_group_sample', params); + const error = resp.hits.hits[0]?._source; + const transactionId = error?.transaction?.id; + const traceId = error?.trace?.id; - let transaction; - if (transactionId && traceId) { - transaction = await getTransaction({ transactionId, traceId, setup }); - } + let transaction; + if (transactionId && traceId) { + transaction = await getTransaction({ transactionId, traceId, setup }); + } - return { - transaction, - error, - occurrencesCount: resp.hits.total.value, - }; - }); + return { + transaction, + error, + occurrencesCount: resp.hits.total.value, + }; } diff --git a/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts b/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts index f5b22e5349756c..d705a2eb5a00c8 100644 --- a/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts +++ b/x-pack/plugins/apm/server/lib/errors/get_error_groups.ts @@ -15,11 +15,10 @@ import { } from '../../../common/elasticsearch_fieldnames'; import { getErrorGroupsProjection } from '../../projections/errors'; import { mergeProjection } from '../../projections/util/merge_projection'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getErrorName } from '../helpers/get_error_name'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -export function getErrorGroups({ +export async function getErrorGroups({ environment, kuery, serviceName, @@ -34,87 +33,83 @@ export function getErrorGroups({ sortDirection?: 'asc' | 'desc'; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_error_groups', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - // sort buckets by last occurrence of error - const sortByLatestOccurrence = sortField === 'latestOccurrenceAt'; + // sort buckets by last occurrence of error + const sortByLatestOccurrence = sortField === 'latestOccurrenceAt'; - const projection = getErrorGroupsProjection({ - environment, - kuery, - setup, - serviceName, - }); + const projection = getErrorGroupsProjection({ + environment, + kuery, + setup, + serviceName, + }); - const order = sortByLatestOccurrence - ? { - max_timestamp: sortDirection, - } - : { _count: sortDirection }; + const order = sortByLatestOccurrence + ? { + max_timestamp: sortDirection, + } + : { _count: sortDirection }; - const params = mergeProjection(projection, { - body: { - size: 0, - aggs: { - error_groups: { - terms: { - ...projection.body.aggs.error_groups.terms, - size: 500, - order, - }, - aggs: { - sample: { - top_hits: { - _source: [ - ERROR_LOG_MESSAGE, - ERROR_EXC_MESSAGE, - ERROR_EXC_HANDLED, - ERROR_EXC_TYPE, - ERROR_CULPRIT, - ERROR_GROUP_ID, - '@timestamp', - ], - sort: [{ '@timestamp': 'desc' as const }], - size: 1, - }, + const params = mergeProjection(projection, { + body: { + size: 0, + aggs: { + error_groups: { + terms: { + ...projection.body.aggs.error_groups.terms, + size: 500, + order, + }, + aggs: { + sample: { + top_hits: { + _source: [ + ERROR_LOG_MESSAGE, + ERROR_EXC_MESSAGE, + ERROR_EXC_HANDLED, + ERROR_EXC_TYPE, + ERROR_CULPRIT, + ERROR_GROUP_ID, + '@timestamp', + ], + sort: [{ '@timestamp': 'desc' as const }], + size: 1, }, - ...(sortByLatestOccurrence - ? { - max_timestamp: { - max: { - field: '@timestamp', - }, - }, - } - : {}), }, + ...(sortByLatestOccurrence + ? { + max_timestamp: { + max: { + field: '@timestamp', + }, + }, + } + : {}), }, }, }, - }); + }, + }); - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search('get_error_groups', params); - // aggregations can be undefined when no matching indices are found. - // this is an exception rather than the rule so the ES type does not account for this. - const hits = (resp.aggregations?.error_groups.buckets || []).map( - (bucket) => { - const source = bucket.sample.hits.hits[0]._source; - const message = getErrorName(source); + // aggregations can be undefined when no matching indices are found. + // this is an exception rather than the rule so the ES type does not account for this. + const hits = (resp.aggregations?.error_groups.buckets || []).map((bucket) => { + const source = bucket.sample.hits.hits[0]._source; + const message = getErrorName(source); - return { - message, - occurrenceCount: bucket.doc_count, - culprit: source.error.culprit, - groupId: source.error.grouping_key, - latestOccurrenceAt: source['@timestamp'], - handled: source.error.exception?.[0].handled, - type: source.error.exception?.[0].type, - }; - } - ); - - return hits; + return { + message, + occurrenceCount: bucket.doc_count, + culprit: source.error.culprit, + groupId: source.error.grouping_key, + latestOccurrenceAt: source['@timestamp'], + handled: source.error.exception?.[0].handled, + type: source.error.exception?.[0].type, + }; }); + + return hits; } diff --git a/x-pack/plugins/apm/server/lib/helpers/aggregated_transactions/index.ts b/x-pack/plugins/apm/server/lib/helpers/aggregated_transactions/index.ts index 394cf6b988f120..8bfb137c1689cf 100644 --- a/x-pack/plugins/apm/server/lib/helpers/aggregated_transactions/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/aggregated_transactions/index.ts @@ -14,7 +14,6 @@ import { } from '../../../../common/elasticsearch_fieldnames'; import { APMConfig } from '../../..'; import { APMEventClient } from '../create_es_client/create_apm_event_client'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function getHasAggregatedTransactions({ start, @@ -25,8 +24,9 @@ export async function getHasAggregatedTransactions({ end?: number; apmEventClient: APMEventClient; }) { - return withApmSpan('get_has_aggregated_transactions', async () => { - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_has_aggregated_transactions', + { apm: { events: [ProcessorEvent.metric], }, @@ -41,14 +41,14 @@ export async function getHasAggregatedTransactions({ }, }, terminateAfter: 1, - }); - - if (response.hits.total.value > 0) { - return true; } + ); + + if (response.hits.total.value > 0) { + return true; + } - return false; - }); + return false; } export async function getSearchAggregatedTransactions({ diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/call_async_with_debug.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/call_async_with_debug.ts index 989297544c78fe..39018e26f371ce 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/call_async_with_debug.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/call_async_with_debug.ts @@ -81,17 +81,26 @@ export async function callAsyncWithDebug({ return res; } -export const getDebugBody = ( - params: Record, - requestType: string -) => { +export const getDebugBody = ({ + params, + requestType, + operationName, +}: { + params: Record; + requestType: string; + operationName: string; +}) => { + const operationLine = `${operationName}\n`; + if (requestType === 'search') { - return `GET ${params.index}/_search\n${formatObj(params.body)}`; + return `${operationLine}GET ${params.index}/_search\n${formatObj( + params.body + )}`; } return `${chalk.bold('ES operation:')} ${requestType}\n${chalk.bold( 'ES query:' - )}\n${formatObj(params)}`; + )}\n${operationLine}${formatObj(params)}`; }; export const getDebugTitle = (request: KibanaRequest) => diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts index addd7391d782d9..8e82a189d75f35 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.test.ts @@ -47,7 +47,7 @@ describe('createApmEventClient', () => { }, }); - await eventClient.search({ + await eventClient.search('foo', { apm: { events: [], }, diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts index b8a14253a229a2..916a6981f286a9 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_apm_event_client/index.ts @@ -6,6 +6,7 @@ */ import { ValuesType } from 'utility-types'; +import { withApmSpan } from '../../../../utils/with_apm_span'; import { Profile } from '../../../../../typings/es_schemas/ui/profile'; import { ElasticsearchClient, @@ -34,6 +35,7 @@ import { unpackProcessorEvents } from './unpack_processor_events'; export type APMEventESSearchRequest = Omit & { apm: { events: ProcessorEvent[]; + includeLegacyData?: boolean; }; }; @@ -78,11 +80,13 @@ export function createApmEventClient({ }) { return { async search( - params: TParams, - { includeLegacyData = false } = {} + operationName: string, + params: TParams ): Promise> { const withProcessorEventFilter = unpackProcessorEvents(params, indices); + const { includeLegacyData = false } = params.apm; + const withPossibleLegacyDataFilter = !includeLegacyData ? addFilterToExcludeLegacyData(withProcessorEventFilter) : withProcessorEventFilter; @@ -98,15 +102,18 @@ export function createApmEventClient({ return callAsyncWithDebug({ cb: () => { - const searchPromise = cancelEsRequestOnAbort( - esClient.search(searchParams), - request + const searchPromise = withApmSpan(operationName, () => + cancelEsRequestOnAbort(esClient.search(searchParams), request) ); return unwrapEsResponse(searchPromise); }, getDebugMessage: () => ({ - body: getDebugBody(searchParams, requestType), + body: getDebugBody({ + params: searchParams, + requestType, + operationName, + }), title: getDebugTitle(request), }), isCalledWithInternalUser: false, diff --git a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_internal_es_client/index.ts b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_internal_es_client/index.ts index 1544538de74a65..e6b61a709ae353 100644 --- a/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_internal_es_client/index.ts +++ b/x-pack/plugins/apm/server/lib/helpers/create_es_client/create_internal_es_client/index.ts @@ -31,20 +31,23 @@ export function createInternalESClient({ }: Pick & { debug: boolean }) { const { asInternalUser } = context.core.elasticsearch.client; - function callEs({ - cb, - requestType, - params, - }: { - requestType: string; - cb: () => TransportRequestPromise; - params: Record; - }) { + function callEs( + operationName: string, + { + cb, + requestType, + params, + }: { + requestType: string; + cb: () => TransportRequestPromise; + params: Record; + } + ) { return callAsyncWithDebug({ cb: () => unwrapEsResponse(cancelEsRequestOnAbort(cb(), request)), getDebugMessage: () => ({ title: getDebugTitle(request), - body: getDebugBody(params, requestType), + body: getDebugBody({ params, requestType, operationName }), }), debug, isCalledWithInternalUser: true, @@ -59,30 +62,37 @@ export function createInternalESClient({ TDocument = unknown, TSearchRequest extends ESSearchRequest = ESSearchRequest >( + operationName: string, params: TSearchRequest ): Promise> => { - return callEs({ + return callEs(operationName, { requestType: 'search', cb: () => asInternalUser.search(params), params, }); }, - index: (params: APMIndexDocumentParams) => { - return callEs({ + index: (operationName: string, params: APMIndexDocumentParams) => { + return callEs(operationName, { requestType: 'index', cb: () => asInternalUser.index(params), params, }); }, - delete: (params: estypes.DeleteRequest): Promise<{ result: string }> => { - return callEs({ + delete: ( + operationName: string, + params: estypes.DeleteRequest + ): Promise<{ result: string }> => { + return callEs(operationName, { requestType: 'delete', cb: () => asInternalUser.delete(params), params, }); }, - indicesCreate: (params: estypes.IndicesCreateRequest) => { - return callEs({ + indicesCreate: ( + operationName: string, + params: estypes.IndicesCreateRequest + ) => { + return callEs(operationName, { requestType: 'indices.create', cb: () => asInternalUser.indices.create(params), params, diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts index c0ff0cab88f47f..66b3c91fc6f2d1 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts @@ -109,7 +109,7 @@ describe('setupRequest', () => { it('calls callWithRequest', async () => { const mockResources = getMockResources(); const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [ProcessorEvent.transaction] }, body: { foo: 'bar' }, }); @@ -137,7 +137,7 @@ describe('setupRequest', () => { it('calls callWithInternalUser', async () => { const mockResources = getMockResources(); const { internalClient } = await setupRequest(mockResources); - await internalClient.search({ + await internalClient.search('foo', { index: ['apm-*'], body: { foo: 'bar' }, } as any); @@ -156,7 +156,7 @@ describe('setupRequest', () => { it('adds a range filter for `observer.version_major` to the existing filter', async () => { const mockResources = getMockResources(); const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [ProcessorEvent.transaction], }, @@ -183,19 +183,15 @@ describe('setupRequest', () => { it('does not add a range filter for `observer.version_major` if includeLegacyData=true', async () => { const mockResources = getMockResources(); const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search( - { - apm: { - events: [ProcessorEvent.error], - }, - body: { - query: { bool: { filter: [{ term: { field: 'someTerm' } }] } }, - }, - }, - { + await apmEventClient.search('foo', { + apm: { + events: [ProcessorEvent.error], includeLegacyData: true, - } - ); + }, + body: { + query: { bool: { filter: [{ term: { field: 'someTerm' } }] } }, + }, + }); const params = mockResources.context.core.elasticsearch.client.asCurrentUser.search .mock.calls[0][0]; @@ -221,7 +217,7 @@ describe('without a bool filter', () => { it('adds a range filter for `observer.version_major`', async () => { const mockResources = getMockResources(); const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [ProcessorEvent.error], }, @@ -251,7 +247,7 @@ describe('with includeFrozen=false', () => { const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [], }, @@ -273,7 +269,7 @@ describe('with includeFrozen=true', () => { const { apmEventClient } = await setupRequest(mockResources); - await apmEventClient.search({ + await apmEventClient.search('foo', { apm: { events: [] }, }); diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts index 3b3ef8b9c4bcf5..d1040b49dcd8b7 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/fetch_and_transform_gc_metrics.ts @@ -30,6 +30,7 @@ export async function fetchAndTransformGcMetrics({ serviceNodeName, chartBase, fieldName, + operationName, }: { environment?: string; kuery?: string; @@ -38,6 +39,7 @@ export async function fetchAndTransformGcMetrics({ serviceNodeName?: string; chartBase: ChartBase; fieldName: typeof METRIC_JAVA_GC_COUNT | typeof METRIC_JAVA_GC_TIME; + operationName: string; }) { const { start, end, apmEventClient, config } = setup; @@ -108,7 +110,7 @@ export async function fetchAndTransformGcMetrics({ }, }); - const response = await apmEventClient.search(params); + const response = await apmEventClient.search(operationName, params); const { aggregations } = response; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts index 388331f3bbf179..3ec40d51716941 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_rate_chart.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_GC_COUNT } from '../../../../../../common/elasticsearch_fieldnames'; import { Setup, SetupTimeRange } from '../../../../helpers/setup_request'; import { fetchAndTransformGcMetrics } from './fetch_and_transform_gc_metrics'; @@ -45,17 +44,16 @@ function getGcRateChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_gc_rate_charts', () => - fetchAndTransformGcMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - fieldName: METRIC_JAVA_GC_COUNT, - }) - ); + return fetchAndTransformGcMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + fieldName: METRIC_JAVA_GC_COUNT, + operationName: 'get_gc_rate_charts', + }); } export { getGcRateChart }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts index e6f80190d1daa3..8e4416d94fb908 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/gc/get_gc_time_chart.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_GC_TIME } from '../../../../../../common/elasticsearch_fieldnames'; import { Setup, SetupTimeRange } from '../../../../helpers/setup_request'; import { fetchAndTransformGcMetrics } from './fetch_and_transform_gc_metrics'; @@ -45,17 +44,16 @@ function getGcTimeChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_gc_time_charts', () => - fetchAndTransformGcMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - fieldName: METRIC_JAVA_GC_TIME, - }) - ); + return fetchAndTransformGcMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + fieldName: METRIC_JAVA_GC_TIME, + operationName: 'get_gc_time_charts', + }); } export { getGcTimeChart }; diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts index 7630827a3cb389..6a23213e94537d 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/heap_memory/index.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_HEAP_MEMORY_MAX, METRIC_JAVA_HEAP_MEMORY_COMMITTED, @@ -65,22 +64,21 @@ export function getHeapMemoryChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_heap_memory_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - heapMemoryMax: { avg: { field: METRIC_JAVA_HEAP_MEMORY_MAX } }, - heapMemoryCommitted: { - avg: { field: METRIC_JAVA_HEAP_MEMORY_COMMITTED }, - }, - heapMemoryUsed: { avg: { field: METRIC_JAVA_HEAP_MEMORY_USED } }, + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + heapMemoryMax: { avg: { field: METRIC_JAVA_HEAP_MEMORY_MAX } }, + heapMemoryCommitted: { + avg: { field: METRIC_JAVA_HEAP_MEMORY_COMMITTED }, }, - additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], - }) - ); + heapMemoryUsed: { avg: { field: METRIC_JAVA_HEAP_MEMORY_USED } }, + }, + additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], + operationName: 'get_heap_memory_charts', + }); } diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts index cd11e5e5383b63..1ceb42b7db4799 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/non_heap_memory/index.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_NON_HEAP_MEMORY_MAX, METRIC_JAVA_NON_HEAP_MEMORY_COMMITTED, @@ -62,24 +61,23 @@ export async function getNonHeapMemoryChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_non_heap_memory_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - nonHeapMemoryMax: { avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_MAX } }, - nonHeapMemoryCommitted: { - avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_COMMITTED }, - }, - nonHeapMemoryUsed: { - avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_USED }, - }, + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + nonHeapMemoryMax: { avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_MAX } }, + nonHeapMemoryCommitted: { + avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_COMMITTED }, }, - additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], - }) - ); + nonHeapMemoryUsed: { + avg: { field: METRIC_JAVA_NON_HEAP_MEMORY_USED }, + }, + }, + additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], + operationName: 'get_non_heap_memory_charts', + }); } diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts index 8d4c079197d191..700c5e08d4defa 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/java/thread_count/index.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_JAVA_THREAD_COUNT, AGENT_NAME, @@ -54,19 +53,18 @@ export async function getThreadCountChart({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_thread_count_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - threadCount: { avg: { field: METRIC_JAVA_THREAD_COUNT } }, - threadCountMax: { max: { field: METRIC_JAVA_THREAD_COUNT } }, - }, - additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], - }) - ); + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + threadCount: { avg: { field: METRIC_JAVA_THREAD_COUNT } }, + threadCountMax: { max: { field: METRIC_JAVA_THREAD_COUNT } }, + }, + additionalFilters: [{ term: { [AGENT_NAME]: 'java' } }], + operationName: 'get_thread_count_charts', + }); } diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts index 37bef191ae876e..a568d58bdd4389 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/cpu/index.ts @@ -7,7 +7,6 @@ import theme from '@elastic/eui/dist/eui_theme_light.json'; import { i18n } from '@kbn/i18n'; -import { withApmSpan } from '../../../../../utils/with_apm_span'; import { METRIC_SYSTEM_CPU_PERCENT, METRIC_PROCESS_CPU_PERCENT, @@ -66,20 +65,19 @@ export function getCPUChartData({ serviceName: string; serviceNodeName?: string; }) { - return withApmSpan('get_cpu_metric_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - systemCPUAverage: { avg: { field: METRIC_SYSTEM_CPU_PERCENT } }, - systemCPUMax: { max: { field: METRIC_SYSTEM_CPU_PERCENT } }, - processCPUAverage: { avg: { field: METRIC_PROCESS_CPU_PERCENT } }, - processCPUMax: { max: { field: METRIC_PROCESS_CPU_PERCENT } }, - }, - }) - ); + return fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + systemCPUAverage: { avg: { field: METRIC_SYSTEM_CPU_PERCENT } }, + systemCPUMax: { max: { field: METRIC_SYSTEM_CPU_PERCENT } }, + processCPUAverage: { avg: { field: METRIC_PROCESS_CPU_PERCENT } }, + processCPUMax: { max: { field: METRIC_PROCESS_CPU_PERCENT } }, + }, + operationName: 'get_cpu_metric_charts', + }); } diff --git a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts index 0ec2f2c2fcfb25..1f7860d567b037 100644 --- a/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts +++ b/x-pack/plugins/apm/server/lib/metrics/by_agent/shared/memory/index.ts @@ -84,45 +84,41 @@ export async function getMemoryChartData({ serviceNodeName?: string; }) { return withApmSpan('get_memory_metrics_charts', async () => { - const cgroupResponse = await withApmSpan( - 'get_cgroup_memory_metrics_charts', - () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - memoryUsedAvg: { avg: { script: percentCgroupMemoryUsedScript } }, - memoryUsedMax: { max: { script: percentCgroupMemoryUsedScript } }, - }, - additionalFilters: [ - { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }, - ], - }) - ); + const cgroupResponse = await fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + memoryUsedAvg: { avg: { script: percentCgroupMemoryUsedScript } }, + memoryUsedMax: { max: { script: percentCgroupMemoryUsedScript } }, + }, + additionalFilters: [ + { exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES } }, + ], + operationName: 'get_cgroup_memory_metrics_charts', + }); if (cgroupResponse.noHits) { - return await withApmSpan('get_system_memory_metrics_charts', () => - fetchAndTransformMetrics({ - environment, - kuery, - setup, - serviceName, - serviceNodeName, - chartBase, - aggs: { - memoryUsedAvg: { avg: { script: percentSystemMemoryUsedScript } }, - memoryUsedMax: { max: { script: percentSystemMemoryUsedScript } }, - }, - additionalFilters: [ - { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, - { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, - ], - }) - ); + return await fetchAndTransformMetrics({ + environment, + kuery, + setup, + serviceName, + serviceNodeName, + chartBase, + aggs: { + memoryUsedAvg: { avg: { script: percentSystemMemoryUsedScript } }, + memoryUsedMax: { max: { script: percentSystemMemoryUsedScript } }, + }, + additionalFilters: [ + { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, + { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + ], + operationName: 'get_system_memory_metrics_charts', + }); } return cgroupResponse; diff --git a/x-pack/plugins/apm/server/lib/metrics/fetch_and_transform_metrics.ts b/x-pack/plugins/apm/server/lib/metrics/fetch_and_transform_metrics.ts index 30234447821ec9..df9e33e6f4b409 100644 --- a/x-pack/plugins/apm/server/lib/metrics/fetch_and_transform_metrics.ts +++ b/x-pack/plugins/apm/server/lib/metrics/fetch_and_transform_metrics.ts @@ -56,6 +56,7 @@ export async function fetchAndTransformMetrics({ chartBase, aggs, additionalFilters = [], + operationName, }: { environment?: string; kuery?: string; @@ -65,6 +66,7 @@ export async function fetchAndTransformMetrics({ chartBase: ChartBase; aggs: T; additionalFilters?: Filter[]; + operationName: string; }) { const { start, end, apmEventClient, config } = setup; @@ -98,7 +100,7 @@ export async function fetchAndTransformMetrics({ }, }); - const response = await apmEventClient.search(params); + const response = await apmEventClient.search(operationName, params); return transformDataToMetricsChart(response, chartBase); } diff --git a/x-pack/plugins/apm/server/lib/observability_overview/get_service_count.ts b/x-pack/plugins/apm/server/lib/observability_overview/get_service_count.ts index 2ccbe318862f1f..086516371387e6 100644 --- a/x-pack/plugins/apm/server/lib/observability_overview/get_service_count.ts +++ b/x-pack/plugins/apm/server/lib/observability_overview/get_service_count.ts @@ -10,40 +10,40 @@ import { rangeQuery } from '../../../server/utils/queries'; import { SERVICE_NAME } from '../../../common/elasticsearch_fieldnames'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getServiceCount({ +export async function getServiceCount({ setup, searchAggregatedTransactions, }: { setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('observability_overview_get_service_count', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - size: 0, - query: { - bool: { - filter: rangeQuery(start, end), - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + size: 0, + query: { + bool: { + filter: rangeQuery(start, end), }, - aggs: { serviceCount: { cardinality: { field: SERVICE_NAME } } }, }, - }; + aggs: { serviceCount: { cardinality: { field: SERVICE_NAME } } }, + }, + }; - const { aggregations } = await apmEventClient.search(params); - return aggregations?.serviceCount.value || 0; - }); + const { aggregations } = await apmEventClient.search( + 'observability_overview_get_service_count', + params + ); + return aggregations?.serviceCount.value || 0; } diff --git a/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts b/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts index da8ac7c50b5947..016cb50566da0d 100644 --- a/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts +++ b/x-pack/plugins/apm/server/lib/observability_overview/get_transactions_per_minute.ts @@ -14,9 +14,8 @@ import { rangeQuery } from '../../../server/utils/queries'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { calculateThroughput } from '../helpers/calculate_throughput'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getTransactionsPerMinute({ +export async function getTransactionsPerMinute({ setup, bucketSize, searchAggregatedTransactions, @@ -25,71 +24,69 @@ export function getTransactionsPerMinute({ bucketSize: string; searchAggregatedTransactions: boolean; }) { - return withApmSpan( - 'observability_overview_get_transactions_per_minute', - async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const { aggregations } = await apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], + const { aggregations } = await apmEventClient.search( + 'observability_overview_get_transactions_per_minute', + { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: rangeQuery(start, end), + }, }, - body: { - size: 0, - query: { - bool: { - filter: rangeQuery(start, end), + aggs: { + transactionType: { + terms: { + field: TRANSACTION_TYPE, }, - }, - aggs: { - transactionType: { - terms: { - field: TRANSACTION_TYPE, - }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: bucketSize, - min_doc_count: 0, - }, - aggs: { - throughput: { rate: { unit: 'minute' as const } }, - }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: bucketSize, + min_doc_count: 0, + }, + aggs: { + throughput: { rate: { unit: 'minute' as const } }, }, }, }, }, }, - }); + }, + } + ); - if (!aggregations || !aggregations.transactionType.buckets) { - return { value: undefined, timeseries: [] }; - } + if (!aggregations || !aggregations.transactionType.buckets) { + return { value: undefined, timeseries: [] }; + } - const topTransactionTypeBucket = - aggregations.transactionType.buckets.find( - ({ key: transactionType }) => - transactionType === TRANSACTION_REQUEST || - transactionType === TRANSACTION_PAGE_LOAD - ) || aggregations.transactionType.buckets[0]; + const topTransactionTypeBucket = + aggregations.transactionType.buckets.find( + ({ key: transactionType }) => + transactionType === TRANSACTION_REQUEST || + transactionType === TRANSACTION_PAGE_LOAD + ) || aggregations.transactionType.buckets[0]; - return { - value: calculateThroughput({ - start, - end, - value: topTransactionTypeBucket?.doc_count || 0, - }), - timeseries: - topTransactionTypeBucket?.timeseries.buckets.map((bucket) => ({ - x: bucket.key, - y: bucket.throughput.value, - })) || [], - }; - } - ); + return { + value: calculateThroughput({ + start, + end, + value: topTransactionTypeBucket?.doc_count || 0, + }), + timeseries: + topTransactionTypeBucket?.timeseries.buckets.map((bucket) => ({ + x: bucket.key, + y: bucket.throughput.value, + })) || [], + }; } diff --git a/x-pack/plugins/apm/server/lib/observability_overview/has_data.ts b/x-pack/plugins/apm/server/lib/observability_overview/has_data.ts index bbe13874d7d3b4..5c1a33e750e12f 100644 --- a/x-pack/plugins/apm/server/lib/observability_overview/has_data.ts +++ b/x-pack/plugins/apm/server/lib/observability_overview/has_data.ts @@ -6,31 +6,31 @@ */ import { ProcessorEvent } from '../../../common/processor_event'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup } from '../helpers/setup_request'; -export function getHasData({ setup }: { setup: Setup }) { - return withApmSpan('observability_overview_has_apm_data', async () => { - const { apmEventClient } = setup; - try { - const params = { - apm: { - events: [ - ProcessorEvent.transaction, - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - terminateAfter: 1, - body: { - size: 0, - }, - }; +export async function getHasData({ setup }: { setup: Setup }) { + const { apmEventClient } = setup; + try { + const params = { + apm: { + events: [ + ProcessorEvent.transaction, + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + terminateAfter: 1, + body: { + size: 0, + }, + }; - const response = await apmEventClient.search(params); - return response.hits.total.value > 0; - } catch (e) { - return false; - } - }); + const response = await apmEventClient.search( + 'observability_overview_has_apm_data', + params + ); + return response.hits.total.value > 0; + } catch (e) { + return false; + } } diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_client_metrics.ts b/x-pack/plugins/apm/server/lib/rum_client/get_client_metrics.ts index ea1a65020e0cca..e56f234c0633ea 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_client_metrics.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_client_metrics.ts @@ -63,7 +63,7 @@ export async function getClientMetrics({ }); const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_client_metrics', params); const { hasFetchStartField: { backEnd, totalPageLoadDuration }, } = response.aggregations!; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_js_errors.ts b/x-pack/plugins/apm/server/lib/rum_client/get_js_errors.ts index d65399f91bef84..6f734a214501d1 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_js_errors.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_js_errors.ts @@ -94,7 +94,7 @@ export async function getJSErrors({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_js_errors', params); const { totalErrorGroups, totalErrorPages, errors } = response.aggregations ?? {}; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_long_task_metrics.ts b/x-pack/plugins/apm/server/lib/rum_client/get_long_task_metrics.ts index c873b9b4aed829..c4c6f613172d14 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_long_task_metrics.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_long_task_metrics.ts @@ -64,7 +64,7 @@ export async function getLongTaskMetrics({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_long_task_metrics', params); const pkey = percentile.toFixed(1); diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_page_load_distribution.ts b/x-pack/plugins/apm/server/lib/rum_client/get_page_load_distribution.ts index 0f1d7146f84596..73d634e3134d18 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_page_load_distribution.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_page_load_distribution.ts @@ -117,7 +117,7 @@ export async function getPageLoadDistribution({ const { aggregations, hits: { total }, - } = await apmEventClient.search(params); + } = await apmEventClient.search('get_page_load_distribution', params); if (total.value === 0) { return null; @@ -210,7 +210,10 @@ const getPercentilesDistribution = async ({ const { apmEventClient } = setup; - const { aggregations } = await apmEventClient.search(params); + const { aggregations } = await apmEventClient.search( + 'get_page_load_distribution', + params + ); return aggregations?.loadDistribution.values ?? []; }; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_page_view_trends.ts b/x-pack/plugins/apm/server/lib/rum_client/get_page_view_trends.ts index 13046e6c5a8738..41af2ae166aaf6 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_page_view_trends.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_page_view_trends.ts @@ -69,7 +69,7 @@ export async function getPageViewTrends({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_page_view_trends', params); const { topBreakdowns } = response.aggregations ?? {}; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_pl_dist_breakdown.ts b/x-pack/plugins/apm/server/lib/rum_client/get_pl_dist_breakdown.ts index 6a6caab9537333..e63d834307a5f0 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_pl_dist_breakdown.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_pl_dist_breakdown.ts @@ -92,7 +92,10 @@ export const getPageLoadDistBreakdown = async ({ const { apmEventClient } = setup; - const { aggregations } = await apmEventClient.search(params); + const { aggregations } = await apmEventClient.search( + 'get_page_load_dist_breakdown', + params + ); const pageDistBreakdowns = aggregations?.breakdowns.buckets; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_rum_services.ts b/x-pack/plugins/apm/server/lib/rum_client/get_rum_services.ts index ffe9225f1ab99a..a2e6b55738d3a1 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_rum_services.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_rum_services.ts @@ -38,7 +38,7 @@ export async function getRumServices({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_rum_services', params); const result = response.aggregations?.services.buckets ?? []; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_url_search.ts b/x-pack/plugins/apm/server/lib/rum_client/get_url_search.ts index 7cf9066fb4d4da..ae65cdbd121ea4 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_url_search.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_url_search.ts @@ -56,7 +56,7 @@ export async function getUrlSearch({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_url_search', params); const { urls, totalUrls } = response.aggregations ?? {}; const pkey = percentile.toFixed(1); diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_visitor_breakdown.ts b/x-pack/plugins/apm/server/lib/rum_client/get_visitor_breakdown.ts index 247b808896e41a..9c7a64d7c6481f 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_visitor_breakdown.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_visitor_breakdown.ts @@ -51,7 +51,7 @@ export async function getVisitorBreakdown({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_visitor_breakdown', params); const { browsers, os } = response.aggregations!; const totalItems = response.hits.total.value; diff --git a/x-pack/plugins/apm/server/lib/rum_client/get_web_core_vitals.ts b/x-pack/plugins/apm/server/lib/rum_client/get_web_core_vitals.ts index 9bde701df56721..bbb301e22aa8de 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/get_web_core_vitals.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/get_web_core_vitals.ts @@ -103,7 +103,7 @@ export async function getWebCoreVitals({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_web_core_vitals', params); const { lcp, cls, diff --git a/x-pack/plugins/apm/server/lib/rum_client/has_rum_data.ts b/x-pack/plugins/apm/server/lib/rum_client/has_rum_data.ts index 87136fc0538a6b..fc5da4ec1d0fa6 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/has_rum_data.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/has_rum_data.ts @@ -51,7 +51,7 @@ export async function hasRumData({ const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('has_rum_data', params); return { indices: setup.indices['apm_oss.transactionIndices']!, hasData: response.hits.total.value > 0, diff --git a/x-pack/plugins/apm/server/lib/rum_client/ui_filters/local_ui_filters/index.ts b/x-pack/plugins/apm/server/lib/rum_client/ui_filters/local_ui_filters/index.ts index 8fdeb77171862b..e0e9bb2ca002f4 100644 --- a/x-pack/plugins/apm/server/lib/rum_client/ui_filters/local_ui_filters/index.ts +++ b/x-pack/plugins/apm/server/lib/rum_client/ui_filters/local_ui_filters/index.ts @@ -38,37 +38,38 @@ export function getLocalUIFilters({ delete projectionWithoutAggs.body.aggs; return Promise.all( - localFilterNames.map(async (name) => - withApmSpan('get_ui_filter_options_for_field', async () => { - const query = getLocalFilterQuery({ - uiFilters, - projection, - localUIFilterName: name, - }); + localFilterNames.map(async (name) => { + const query = getLocalFilterQuery({ + uiFilters, + projection, + localUIFilterName: name, + }); - const response = await apmEventClient.search(query); + const response = await apmEventClient.search( + 'get_ui_filter_options_for_field', + query + ); - const filter = localUIFilters[name]; + const filter = localUIFilters[name]; - const buckets = response?.aggregations?.by_terms?.buckets ?? []; + const buckets = response?.aggregations?.by_terms?.buckets ?? []; - return { - ...filter, - options: orderBy( - buckets.map((bucket) => { - return { - name: bucket.key as string, - count: bucket.bucket_count - ? bucket.bucket_count.value - : bucket.doc_count, - }; - }), - 'count', - 'desc' - ), - }; - }) - ) + return { + ...filter, + options: orderBy( + buckets.map((bucket) => { + return { + name: bucket.key as string, + count: bucket.bucket_count + ? bucket.bucket_count.value + : bucket.doc_count, + }; + }), + 'count', + 'desc' + ), + }; + }) ); }); } diff --git a/x-pack/plugins/apm/server/lib/service_map/fetch_service_paths_from_trace_ids.ts b/x-pack/plugins/apm/server/lib/service_map/fetch_service_paths_from_trace_ids.ts index 6047b97651e6ad..6ecfe425dc8c5a 100644 --- a/x-pack/plugins/apm/server/lib/service_map/fetch_service_paths_from_trace_ids.ts +++ b/x-pack/plugins/apm/server/lib/service_map/fetch_service_paths_from_trace_ids.ts @@ -14,44 +14,42 @@ import { ServiceConnectionNode, } from '../../../common/service_map'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -import { withApmSpan } from '../../utils/with_apm_span'; export async function fetchServicePathsFromTraceIds( setup: Setup & SetupTimeRange, traceIds: string[] ) { - return withApmSpan('get_service_paths_from_trace_ids', async () => { - const { apmEventClient } = setup; - - // make sure there's a range so ES can skip shards - const dayInMs = 24 * 60 * 60 * 1000; - const start = setup.start - dayInMs; - const end = setup.end + dayInMs; - - const serviceMapParams = { - apm: { - events: [ProcessorEvent.span, ProcessorEvent.transaction], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { - terms: { - [TRACE_ID]: traceIds, - }, + const { apmEventClient } = setup; + + // make sure there's a range so ES can skip shards + const dayInMs = 24 * 60 * 60 * 1000; + const start = setup.start - dayInMs; + const end = setup.end + dayInMs; + + const serviceMapParams = { + apm: { + events: [ProcessorEvent.span, ProcessorEvent.transaction], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { + terms: { + [TRACE_ID]: traceIds, }, - ...rangeQuery(start, end), - ], - }, + }, + ...rangeQuery(start, end), + ], }, - aggs: { - service_map: { - scripted_metric: { - init_script: { - lang: 'painless', - source: `state.eventsById = new HashMap(); + }, + aggs: { + service_map: { + scripted_metric: { + init_script: { + lang: 'painless', + source: `state.eventsById = new HashMap(); String[] fieldsToCopy = new String[] { 'parent.id', @@ -65,10 +63,10 @@ export async function fetchServicePathsFromTraceIds( 'agent.name' }; state.fieldsToCopy = fieldsToCopy;`, - }, - map_script: { - lang: 'painless', - source: `def id; + }, + map_script: { + lang: 'painless', + source: `def id; if (!doc['span.id'].empty) { id = doc['span.id'].value; } else { @@ -85,14 +83,14 @@ export async function fetchServicePathsFromTraceIds( } state.eventsById[id] = copy`, - }, - combine_script: { - lang: 'painless', - source: `return state.eventsById;`, - }, - reduce_script: { - lang: 'painless', - source: ` + }, + combine_script: { + lang: 'painless', + source: `return state.eventsById;`, + }, + reduce_script: { + lang: 'painless', + source: ` def getDestination ( def event ) { def destination = new HashMap(); destination['span.destination.service.resource'] = event['span.destination.service.resource']; @@ -208,29 +206,29 @@ export async function fetchServicePathsFromTraceIds( response.discoveredServices = discoveredServices; return response;`, - }, }, }, } as const, }, - }; - - const serviceMapFromTraceIdsScriptResponse = await apmEventClient.search( - serviceMapParams - ); - - return serviceMapFromTraceIdsScriptResponse as { - aggregations?: { - service_map: { - value: { - paths: ConnectionNode[][]; - discoveredServices: Array<{ - from: ExternalConnectionNode; - to: ServiceConnectionNode; - }>; - }; + }, + }; + + const serviceMapFromTraceIdsScriptResponse = await apmEventClient.search( + 'get_service_paths_from_trace_ids', + serviceMapParams + ); + + return serviceMapFromTraceIdsScriptResponse as { + aggregations?: { + service_map: { + value: { + paths: ConnectionNode[][]; + discoveredServices: Array<{ + from: ExternalConnectionNode; + to: ServiceConnectionNode; + }>; }; }; }; - }); + }; } diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts index e5b0b72b8784ae..6d50023d3fd0e7 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_map.ts @@ -87,69 +87,70 @@ async function getConnectionData({ } async function getServicesData(options: IEnvOptions) { - return withApmSpan('get_service_stats_for_service_map', async () => { - const { environment, setup, searchAggregatedTransactions } = options; + const { environment, setup, searchAggregatedTransactions } = options; - const projection = getServicesProjection({ - setup, - searchAggregatedTransactions, - }); + const projection = getServicesProjection({ + setup, + searchAggregatedTransactions, + }); - let filter = [ - ...projection.body.query.bool.filter, - ...environmentQuery(environment), - ]; + let filter = [ + ...projection.body.query.bool.filter, + ...environmentQuery(environment), + ]; - if (options.serviceName) { - filter = filter.concat({ - term: { - [SERVICE_NAME]: options.serviceName, + if (options.serviceName) { + filter = filter.concat({ + term: { + [SERVICE_NAME]: options.serviceName, + }, + }); + } + + const params = mergeProjection(projection, { + body: { + size: 0, + query: { + bool: { + ...projection.body.query.bool, + filter, }, - }); - } - - const params = mergeProjection(projection, { - body: { - size: 0, - query: { - bool: { - ...projection.body.query.bool, - filter, + }, + aggs: { + services: { + terms: { + field: projection.body.aggs.services.terms.field, + size: 500, }, - }, - aggs: { - services: { - terms: { - field: projection.body.aggs.services.terms.field, - size: 500, - }, - aggs: { - agent_name: { - terms: { - field: AGENT_NAME, - }, + aggs: { + agent_name: { + terms: { + field: AGENT_NAME, }, }, }, }, }, - }); + }, + }); - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search( + 'get_service_stats_for_service_map', + params + ); - return ( - response.aggregations?.services.buckets.map((bucket) => { - return { - [SERVICE_NAME]: bucket.key as string, - [AGENT_NAME]: - (bucket.agent_name.buckets[0]?.key as string | undefined) || '', - [SERVICE_ENVIRONMENT]: options.environment || null, - }; - }) || [] - ); - }); + return ( + response.aggregations?.services.buckets.map((bucket) => { + return { + [SERVICE_NAME]: bucket.key as string, + [AGENT_NAME]: + (bucket.agent_name.buckets[0]?.key as string | undefined) || '', + [SERVICE_ENVIRONMENT]: options.environment || null, + }; + }) || [] + ); } export type ConnectionsResponse = PromiseReturnType; diff --git a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts index 9850c36c573dd7..2709fb640d8ce3 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_service_map_service_node_info.ts @@ -120,7 +120,7 @@ async function getErrorStats({ }); } -function getTransactionStats({ +async function getTransactionStats({ setup, filter, minutes, @@ -129,68 +129,70 @@ function getTransactionStats({ avgTransactionDuration: number | null; avgRequestsPerMinute: number | null; }> { - return withApmSpan('get_transaction_stats_for_service_map_node', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - ...filter, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - { - terms: { - [TRANSACTION_TYPE]: [ - TRANSACTION_REQUEST, - TRANSACTION_PAGE_LOAD, - ], - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + ...filter, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + { + terms: { + [TRANSACTION_TYPE]: [ + TRANSACTION_REQUEST, + TRANSACTION_PAGE_LOAD, + ], }, - ], - }, - }, - track_total_hits: true, - aggs: { - duration: { - avg: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), }, + ], + }, + }, + track_total_hits: true, + aggs: { + duration: { + avg: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, }, }, - }; - const response = await apmEventClient.search(params); + }, + }; + const response = await apmEventClient.search( + 'get_transaction_stats_for_service_map_node', + params + ); - const totalRequests = response.hits.total.value; + const totalRequests = response.hits.total.value; - return { - avgTransactionDuration: response.aggregations?.duration.value ?? null, - avgRequestsPerMinute: totalRequests > 0 ? totalRequests / minutes : null, - }; - }); + return { + avgTransactionDuration: response.aggregations?.duration.value ?? null, + avgRequestsPerMinute: totalRequests > 0 ? totalRequests / minutes : null, + }; } -function getCpuStats({ +async function getCpuStats({ setup, filter, }: TaskParameters): Promise<{ avgCpuUsage: number | null }> { - return withApmSpan('get_avg_cpu_usage_for_service_map_node', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_avg_cpu_usage_for_service_map_node', + { apm: { events: [ProcessorEvent.metric], }, @@ -206,10 +208,10 @@ function getCpuStats({ }, aggs: { avgCpuUsage: { avg: { field: METRIC_SYSTEM_CPU_PERCENT } } }, }, - }); + } + ); - return { avgCpuUsage: response.aggregations?.avgCpuUsage.value ?? null }; - }); + return { avgCpuUsage: response.aggregations?.avgCpuUsage.value ?? null }; } function getMemoryStats({ @@ -219,7 +221,7 @@ function getMemoryStats({ return withApmSpan('get_memory_stats_for_service_map_node', async () => { const { apmEventClient } = setup; - const getAvgMemoryUsage = ({ + const getAvgMemoryUsage = async ({ additionalFilters, script, }: { @@ -228,8 +230,9 @@ function getMemoryStats({ | typeof percentCgroupMemoryUsedScript | typeof percentSystemMemoryUsedScript; }) => { - return withApmSpan('get_avg_memory_for_service_map_node', async () => { - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_avg_memory_for_service_map_node', + { apm: { events: [ProcessorEvent.metric], }, @@ -244,9 +247,9 @@ function getMemoryStats({ avgMemoryUsage: { avg: { script } }, }, }, - }); - return response.aggregations?.avgMemoryUsage.value ?? null; - }); + } + ); + return response.aggregations?.avgMemoryUsage.value ?? null; }; let avgMemoryUsage = await getAvgMemoryUsage({ diff --git a/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts b/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts index fa04b963388b20..7894a95cf4d7e4 100644 --- a/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts +++ b/x-pack/plugins/apm/server/lib/service_map/get_trace_sample_ids.ts @@ -18,12 +18,11 @@ import { import { ProcessorEvent } from '../../../common/processor_event'; import { SERVICE_MAP_TIMEOUT_ERROR } from '../../../common/service_map'; import { environmentQuery, rangeQuery } from '../../../server/utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; const MAX_TRACES_TO_INSPECT = 1000; -export function getTraceSampleIds({ +export async function getTraceSampleIds({ serviceName, environment, setup, @@ -32,90 +31,88 @@ export function getTraceSampleIds({ environment?: string; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_trace_sample_ids', async () => { - const { start, end, apmEventClient, config } = setup; + const { start, end, apmEventClient, config } = setup; - const query = { - bool: { - filter: [ - { - exists: { - field: SPAN_DESTINATION_SERVICE_RESOURCE, - }, + const query = { + bool: { + filter: [ + { + exists: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, }, - ...rangeQuery(start, end), - ] as ESFilter[], - }, - } as { bool: { filter: ESFilter[]; must_not?: ESFilter[] | ESFilter } }; + }, + ...rangeQuery(start, end), + ] as ESFilter[], + }, + } as { bool: { filter: ESFilter[]; must_not?: ESFilter[] | ESFilter } }; - if (serviceName) { - query.bool.filter.push({ term: { [SERVICE_NAME]: serviceName } }); - } + if (serviceName) { + query.bool.filter.push({ term: { [SERVICE_NAME]: serviceName } }); + } - query.bool.filter.push(...environmentQuery(environment)); + query.bool.filter.push(...environmentQuery(environment)); - const fingerprintBucketSize = serviceName - ? config['xpack.apm.serviceMapFingerprintBucketSize'] - : config['xpack.apm.serviceMapFingerprintGlobalBucketSize']; + const fingerprintBucketSize = serviceName + ? config['xpack.apm.serviceMapFingerprintBucketSize'] + : config['xpack.apm.serviceMapFingerprintGlobalBucketSize']; - const traceIdBucketSize = serviceName - ? config['xpack.apm.serviceMapTraceIdBucketSize'] - : config['xpack.apm.serviceMapTraceIdGlobalBucketSize']; + const traceIdBucketSize = serviceName + ? config['xpack.apm.serviceMapTraceIdBucketSize'] + : config['xpack.apm.serviceMapTraceIdGlobalBucketSize']; - const samplerShardSize = traceIdBucketSize * 10; + const samplerShardSize = traceIdBucketSize * 10; - const params = { - apm: { - events: [ProcessorEvent.span], - }, - body: { - size: 0, - query, - aggs: { - connections: { - composite: { - sources: asMutableArray([ - { - [SPAN_DESTINATION_SERVICE_RESOURCE]: { - terms: { - field: SPAN_DESTINATION_SERVICE_RESOURCE, - }, + const params = { + apm: { + events: [ProcessorEvent.span], + }, + body: { + size: 0, + query, + aggs: { + connections: { + composite: { + sources: asMutableArray([ + { + [SPAN_DESTINATION_SERVICE_RESOURCE]: { + terms: { + field: SPAN_DESTINATION_SERVICE_RESOURCE, }, }, - { - [SERVICE_NAME]: { - terms: { - field: SERVICE_NAME, - }, + }, + { + [SERVICE_NAME]: { + terms: { + field: SERVICE_NAME, }, }, - { - [SERVICE_ENVIRONMENT]: { - terms: { - field: SERVICE_ENVIRONMENT, - missing_bucket: true, - }, + }, + { + [SERVICE_ENVIRONMENT]: { + terms: { + field: SERVICE_ENVIRONMENT, + missing_bucket: true, }, }, - ] as const), - size: fingerprintBucketSize, - }, - aggs: { - sample: { - sampler: { - shard_size: samplerShardSize, - }, - aggs: { - trace_ids: { - terms: { - field: TRACE_ID, - size: traceIdBucketSize, - execution_hint: 'map' as const, - // remove bias towards large traces by sorting on trace.id - // which will be random-esque - order: { - _key: 'desc' as const, - }, + }, + ] as const), + size: fingerprintBucketSize, + }, + aggs: { + sample: { + sampler: { + shard_size: samplerShardSize, + }, + aggs: { + trace_ids: { + terms: { + field: TRACE_ID, + size: traceIdBucketSize, + execution_hint: 'map' as const, + // remove bias towards large traces by sorting on trace.id + // which will be random-esque + order: { + _key: 'desc' as const, }, }, }, @@ -124,34 +121,36 @@ export function getTraceSampleIds({ }, }, }, - }; + }, + }; - try { - const tracesSampleResponse = await apmEventClient.search(params); - // make sure at least one trace per composite/connection bucket - // is queried - const traceIdsWithPriority = - tracesSampleResponse.aggregations?.connections.buckets.flatMap( - (bucket) => - bucket.sample.trace_ids.buckets.map((sampleDocBucket, index) => ({ - traceId: sampleDocBucket.key as string, - priority: index, - })) - ) || []; + try { + const tracesSampleResponse = await apmEventClient.search( + 'get_trace_sample_ids', + params + ); + // make sure at least one trace per composite/connection bucket + // is queried + const traceIdsWithPriority = + tracesSampleResponse.aggregations?.connections.buckets.flatMap((bucket) => + bucket.sample.trace_ids.buckets.map((sampleDocBucket, index) => ({ + traceId: sampleDocBucket.key as string, + priority: index, + })) + ) || []; - const traceIds = take( - uniq( - sortBy(traceIdsWithPriority, 'priority').map(({ traceId }) => traceId) - ), - MAX_TRACES_TO_INSPECT - ); + const traceIds = take( + uniq( + sortBy(traceIdsWithPriority, 'priority').map(({ traceId }) => traceId) + ), + MAX_TRACES_TO_INSPECT + ); - return { traceIds }; - } catch (error) { - if ('displayName' in error && error.displayName === 'RequestTimeout') { - throw Boom.internal(SERVICE_MAP_TIMEOUT_ERROR); - } - throw error; + return { traceIds }; + } catch (error) { + if ('displayName' in error && error.displayName === 'RequestTimeout') { + throw Boom.internal(SERVICE_MAP_TIMEOUT_ERROR); } - }); + throw error; + } } diff --git a/x-pack/plugins/apm/server/lib/service_nodes/index.ts b/x-pack/plugins/apm/server/lib/service_nodes/index.ts index 07b7e532d80558..97c553f344205a 100644 --- a/x-pack/plugins/apm/server/lib/service_nodes/index.ts +++ b/x-pack/plugins/apm/server/lib/service_nodes/index.ts @@ -14,10 +14,9 @@ import { import { SERVICE_NODE_NAME_MISSING } from '../../../common/service_nodes'; import { getServiceNodesProjection } from '../../projections/service_nodes'; import { mergeProjection } from '../../projections/util/merge_projection'; -import { withApmSpan } from '../../utils/with_apm_span'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -const getServiceNodes = ({ +const getServiceNodes = async ({ kuery, setup, serviceName, @@ -26,69 +25,67 @@ const getServiceNodes = ({ setup: Setup & SetupTimeRange; serviceName: string; }) => { - return withApmSpan('get_service_nodes', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const projection = getServiceNodesProjection({ kuery, setup, serviceName }); + const projection = getServiceNodesProjection({ kuery, setup, serviceName }); - const params = mergeProjection(projection, { - body: { - aggs: { - nodes: { - terms: { - ...projection.body.aggs.nodes.terms, - size: 10000, - missing: SERVICE_NODE_NAME_MISSING, - }, - aggs: { - cpu: { - avg: { - field: METRIC_PROCESS_CPU_PERCENT, - }, + const params = mergeProjection(projection, { + body: { + aggs: { + nodes: { + terms: { + ...projection.body.aggs.nodes.terms, + size: 10000, + missing: SERVICE_NODE_NAME_MISSING, + }, + aggs: { + cpu: { + avg: { + field: METRIC_PROCESS_CPU_PERCENT, }, - heapMemory: { - avg: { - field: METRIC_JAVA_HEAP_MEMORY_USED, - }, + }, + heapMemory: { + avg: { + field: METRIC_JAVA_HEAP_MEMORY_USED, }, - nonHeapMemory: { - avg: { - field: METRIC_JAVA_NON_HEAP_MEMORY_USED, - }, + }, + nonHeapMemory: { + avg: { + field: METRIC_JAVA_NON_HEAP_MEMORY_USED, }, - threadCount: { - max: { - field: METRIC_JAVA_THREAD_COUNT, - }, + }, + threadCount: { + max: { + field: METRIC_JAVA_THREAD_COUNT, }, }, }, }, }, - }); + }, + }); - const response = await apmEventClient.search(params); + const response = await apmEventClient.search('get_service_nodes', params); - if (!response.aggregations) { - return []; - } + if (!response.aggregations) { + return []; + } - return response.aggregations.nodes.buckets - .map((bucket) => ({ - name: bucket.key as string, - cpu: bucket.cpu.value, - heapMemory: bucket.heapMemory.value, - nonHeapMemory: bucket.nonHeapMemory.value, - threadCount: bucket.threadCount.value, - })) - .filter( - (item) => - item.cpu !== null || - item.heapMemory !== null || - item.nonHeapMemory !== null || - item.threadCount != null - ); - }); + return response.aggregations.nodes.buckets + .map((bucket) => ({ + name: bucket.key as string, + cpu: bucket.cpu.value, + heapMemory: bucket.heapMemory.value, + nonHeapMemory: bucket.nonHeapMemory.value, + threadCount: bucket.threadCount.value, + })) + .filter( + (item) => + item.cpu !== null || + item.heapMemory !== null || + item.nonHeapMemory !== null || + item.threadCount != null + ); }; export { getServiceNodes }; diff --git a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap index 9d05369aca840f..2f653e2c4df1db 100644 --- a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap @@ -22,6 +22,7 @@ Object { "events": Array [ "transaction", ], + "includeLegacyData": true, }, "body": Object { "query": Object { diff --git a/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts b/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts index 611f9b18a0b1a2..202b5075d2ea74 100644 --- a/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts +++ b/x-pack/plugins/apm/server/lib/services/annotations/get_derived_service_annotations.ts @@ -13,7 +13,6 @@ import { SERVICE_VERSION, } from '../../../../common/elasticsearch_fieldnames'; import { environmentQuery, rangeQuery } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, @@ -31,20 +30,52 @@ export async function getDerivedServiceAnnotations({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_derived_service_annotations', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const filter: ESFilter[] = [ - { term: { [SERVICE_NAME]: serviceName } }, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - ...environmentQuery(environment), - ]; + const filter: ESFilter[] = [ + { term: { [SERVICE_NAME]: serviceName } }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ...environmentQuery(environment), + ]; - const versions = - ( - await apmEventClient.search({ + const versions = + ( + await apmEventClient.search('get_derived_service_annotations', { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [...filter, ...rangeQuery(start, end)], + }, + }, + aggs: { + versions: { + terms: { + field: SERVICE_VERSION, + }, + }, + }, + }, + }) + ).aggregations?.versions.buckets.map((bucket) => bucket.key) ?? []; + + if (versions.length <= 1) { + return []; + } + const annotations = await Promise.all( + versions.map(async (version) => { + const response = await apmEventClient.search( + 'get_first_seen_of_version', + { apm: { events: [ getProcessorEventForAggregatedTransactions( @@ -53,73 +84,40 @@ export async function getDerivedServiceAnnotations({ ], }, body: { - size: 0, + size: 1, query: { bool: { - filter: [...filter, ...rangeQuery(start, end)], + filter: [...filter, { term: { [SERVICE_VERSION]: version } }], }, }, - aggs: { - versions: { - terms: { - field: SERVICE_VERSION, - }, - }, + sort: { + '@timestamp': 'asc', }, }, - }) - ).aggregations?.versions.buckets.map((bucket) => bucket.key) ?? []; + } + ); - if (versions.length <= 1) { - return []; - } - const annotations = await Promise.all( - versions.map(async (version) => { - return withApmSpan('get_first_seen_of_version', async () => { - const response = await apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - size: 1, - query: { - bool: { - filter: [...filter, { term: { [SERVICE_VERSION]: version } }], - }, - }, - sort: { - '@timestamp': 'asc', - }, - }, - }); - - const firstSeen = new Date( - response.hits.hits[0]._source['@timestamp'] - ).getTime(); + const firstSeen = new Date( + response.hits.hits[0]._source['@timestamp'] + ).getTime(); - if (!isFiniteNumber(firstSeen)) { - throw new Error( - 'First seen for version was unexpectedly undefined or null.' - ); - } + if (!isFiniteNumber(firstSeen)) { + throw new Error( + 'First seen for version was unexpectedly undefined or null.' + ); + } - if (firstSeen < start || firstSeen > end) { - return null; - } + if (firstSeen < start || firstSeen > end) { + return null; + } - return { - type: AnnotationType.VERSION, - id: version, - '@timestamp': firstSeen, - text: version, - }; - }); - }) - ); - return annotations.filter(Boolean) as Annotation[]; - }); + return { + type: AnnotationType.VERSION, + id: version, + '@timestamp': firstSeen, + text: version, + }; + }) + ); + return annotations.filter(Boolean) as Annotation[]; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_agent_name.ts b/x-pack/plugins/apm/server/lib/services/get_service_agent_name.ts index a81c0b2fc2c443..82147d7c94236e 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_agent_name.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_agent_name.ts @@ -13,9 +13,8 @@ import { import { rangeQuery } from '../../../server/utils/queries'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getServiceAgentName({ +export async function getServiceAgentName({ serviceName, setup, searchAggregatedTransactions, @@ -24,42 +23,41 @@ export function getServiceAgentName({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_service_agent_name', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const params = { - terminateAfter: 1, - apm: { - events: [ - ProcessorEvent.error, - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.metric, - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ], - }, + const params = { + terminateAfter: 1, + apm: { + events: [ + ProcessorEvent.error, + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.metric, + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ], }, - aggs: { - agents: { - terms: { field: AGENT_NAME, size: 1 }, - }, + }, + aggs: { + agents: { + terms: { field: AGENT_NAME, size: 1 }, }, }, - }; + }, + }; - const { aggregations } = await apmEventClient.search(params); - const agentName = aggregations?.agents.buckets[0]?.key as - | string - | undefined; - return { agentName }; - }); + const { aggregations } = await apmEventClient.search( + 'get_service_agent_name', + params + ); + const agentName = aggregations?.agents.buckets[0]?.key as string | undefined; + return { agentName }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_destination_map.ts b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_destination_map.ts index db491012c986b1..4993484f5b2400 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_destination_map.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_destination_map.ts @@ -38,56 +38,54 @@ export const getDestinationMap = ({ return withApmSpan('get_service_destination_map', async () => { const { start, end, apmEventClient } = setup; - const response = await withApmSpan('get_exit_span_samples', async () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.span], + const response = await apmEventClient.search('get_exit_span_samples', { + apm: { + events: [ProcessorEvent.span], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { exists: { field: SPAN_DESTINATION_SERVICE_RESOURCE } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ], + }, }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { exists: { field: SPAN_DESTINATION_SERVICE_RESOURCE } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ], + aggs: { + connections: { + composite: { + size: 1000, + sources: asMutableArray([ + { + [SPAN_DESTINATION_SERVICE_RESOURCE]: { + terms: { field: SPAN_DESTINATION_SERVICE_RESOURCE }, + }, + }, + // make sure we get samples for both successful + // and failed calls + { [EVENT_OUTCOME]: { terms: { field: EVENT_OUTCOME } } }, + ] as const), }, - }, - aggs: { - connections: { - composite: { - size: 1000, - sources: asMutableArray([ - { - [SPAN_DESTINATION_SERVICE_RESOURCE]: { - terms: { field: SPAN_DESTINATION_SERVICE_RESOURCE }, + aggs: { + sample: { + top_hits: { + size: 1, + _source: [SPAN_TYPE, SPAN_SUBTYPE, SPAN_ID], + sort: [ + { + '@timestamp': 'desc' as const, }, - }, - // make sure we get samples for both successful - // and failed calls - { [EVENT_OUTCOME]: { terms: { field: EVENT_OUTCOME } } }, - ] as const), - }, - aggs: { - sample: { - top_hits: { - size: 1, - _source: [SPAN_TYPE, SPAN_SUBTYPE, SPAN_ID], - sort: [ - { - '@timestamp': 'desc' as const, - }, - ], - }, + ], }, }, }, }, }, - }) - ); + }, + }); const outgoingConnections = response.aggregations?.connections.buckets.map((bucket) => { @@ -103,38 +101,37 @@ export const getDestinationMap = ({ }; }) ?? []; - const transactionResponse = await withApmSpan( + const transactionResponse = await apmEventClient.search( 'get_transactions_for_exit_spans', - () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.transaction], - }, - body: { - query: { - bool: { - filter: [ - { - terms: { - [PARENT_ID]: outgoingConnections.map( - (connection) => connection[SPAN_ID] - ), - }, + { + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + query: { + bool: { + filter: [ + { + terms: { + [PARENT_ID]: outgoingConnections.map( + (connection) => connection[SPAN_ID] + ), }, - ...rangeQuery(start, end), - ], - }, + }, + ...rangeQuery(start, end), + ], }, - size: outgoingConnections.length, - docvalue_fields: asMutableArray([ - SERVICE_NAME, - SERVICE_ENVIRONMENT, - AGENT_NAME, - PARENT_ID, - ] as const), - _source: false, }, - }) + size: outgoingConnections.length, + docvalue_fields: asMutableArray([ + SERVICE_NAME, + SERVICE_ENVIRONMENT, + AGENT_NAME, + PARENT_ID, + ] as const), + _source: false, + }, + } ); const incomingConnections = transactionResponse.hits.hits.map((hit) => ({ diff --git a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_metrics.ts b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_metrics.ts index c8642c6272b5f4..1d815dd7180e3f 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_metrics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_dependencies/get_metrics.ts @@ -18,9 +18,8 @@ import { environmentQuery, rangeQuery } from '../../../../server/utils/queries'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { EventOutcome } from '../../../../common/event_outcome'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export const getMetrics = ({ +export const getMetrics = async ({ setup, serviceName, environment, @@ -31,10 +30,11 @@ export const getMetrics = ({ environment?: string; numBuckets: number; }) => { - return withApmSpan('get_service_destination_metrics', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_destination_metrics', + { apm: { events: [ProcessorEvent.metric], }, @@ -46,7 +46,9 @@ export const getMetrics = ({ filter: [ { term: { [SERVICE_NAME]: serviceName } }, { - exists: { field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT }, + exists: { + field: SPAN_DESTINATION_SERVICE_RESPONSE_TIME_COUNT, + }, }, ...rangeQuery(start, end), ...environmentQuery(environment), @@ -99,47 +101,47 @@ export const getMetrics = ({ }, }, }, - }); + } + ); - return ( - response.aggregations?.connections.buckets.map((bucket) => ({ - span: { - destination: { - service: { - resource: String(bucket.key), - }, + return ( + response.aggregations?.connections.buckets.map((bucket) => ({ + span: { + destination: { + service: { + resource: String(bucket.key), }, }, - value: { - count: sum( - bucket.timeseries.buckets.map( - (dateBucket) => dateBucket.count.value ?? 0 - ) - ), - latency_sum: sum( - bucket.timeseries.buckets.map( - (dateBucket) => dateBucket.latency_sum.value ?? 0 - ) - ), - error_count: sum( - bucket.timeseries.buckets.flatMap( - (dateBucket) => - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0 - ) - ), - }, - timeseries: bucket.timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - count: dateBucket.count.value ?? 0, - latency_sum: dateBucket.latency_sum.value ?? 0, - error_count: - dateBucket[EVENT_OUTCOME].buckets.find( - (outcomeBucket) => outcomeBucket.key === EventOutcome.failure - )?.count.value ?? 0, - })), - })) ?? [] - ); - }); + }, + value: { + count: sum( + bucket.timeseries.buckets.map( + (dateBucket) => dateBucket.count.value ?? 0 + ) + ), + latency_sum: sum( + bucket.timeseries.buckets.map( + (dateBucket) => dateBucket.latency_sum.value ?? 0 + ) + ), + error_count: sum( + bucket.timeseries.buckets.flatMap( + (dateBucket) => + dateBucket[EVENT_OUTCOME].buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.failure + )?.count.value ?? 0 + ) + ), + }, + timeseries: bucket.timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + count: dateBucket.count.value ?? 0, + latency_sum: dateBucket.latency_sum.value ?? 0, + error_count: + dateBucket[EVENT_OUTCOME].buckets.find( + (outcomeBucket) => outcomeBucket.key === EventOutcome.failure + )?.count.value ?? 0, + })), + })) ?? [] + ); }; diff --git a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_detailed_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_detailed_statistics.ts index e45864de2fc1e5..bd69bfc53db71f 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_detailed_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_detailed_statistics.ts @@ -18,7 +18,6 @@ import { rangeQuery, kqlQuery, } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getBucketSize } from '../../helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; @@ -43,75 +42,71 @@ export async function getServiceErrorGroupDetailedStatistics({ start: number; end: number; }): Promise> { - return withApmSpan( - 'get_service_error_group_detailed_statistics', - async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets }); + const { intervalString } = getBucketSize({ start, end, numBuckets }); - const timeseriesResponse = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.error], + const timeseriesResponse = await apmEventClient.search( + 'get_service_error_group_detailed_statistics', + { + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { terms: { [ERROR_GROUP_ID]: groupIds } }, + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + }, }, - body: { - size: 0, - query: { - bool: { - filter: [ - { terms: { [ERROR_GROUP_ID]: groupIds } }, - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], + aggs: { + error_groups: { + terms: { + field: ERROR_GROUP_ID, + size: 500, }, - }, - aggs: { - error_groups: { - terms: { - field: ERROR_GROUP_ID, - size: 500, - }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { - min: start, - max: end, - }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, }, }, }, }, }, }, - }); - - if (!timeseriesResponse.aggregations) { - return []; - } - - return timeseriesResponse.aggregations.error_groups.buckets.map( - (bucket) => { - const groupId = bucket.key as string; - return { - groupId, - timeseries: bucket.timeseries.buckets.map((timeseriesBucket) => { - return { - x: timeseriesBucket.key, - y: timeseriesBucket.doc_count, - }; - }), - }; - } - ); + }, } ); + + if (!timeseriesResponse.aggregations) { + return []; + } + + return timeseriesResponse.aggregations.error_groups.buckets.map((bucket) => { + const groupId = bucket.key as string; + return { + groupId, + timeseries: bucket.timeseries.buckets.map((timeseriesBucket) => { + return { + x: timeseriesBucket.key, + y: timeseriesBucket.doc_count, + }; + }), + }; + }); } export async function getServiceErrorGroupPeriods({ diff --git a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_main_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_main_statistics.ts index 361c92244aee06..8168c0d5549aa4 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_main_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/get_service_error_group_main_statistics.ts @@ -19,11 +19,10 @@ import { rangeQuery, kqlQuery, } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getErrorName } from '../../helpers/get_error_name'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -export function getServiceErrorGroupMainStatistics({ +export async function getServiceErrorGroupMainStatistics({ kuery, serviceName, setup, @@ -36,10 +35,11 @@ export function getServiceErrorGroupMainStatistics({ transactionType: string; environment?: string; }) { - return withApmSpan('get_service_error_group_main_statistics', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_error_group_main_statistics', + { apm: { events: [ProcessorEvent.error], }, @@ -79,24 +79,23 @@ export function getServiceErrorGroupMainStatistics({ }, }, }, - }); + } + ); - const errorGroups = - response.aggregations?.error_groups.buckets.map((bucket) => ({ - group_id: bucket.key as string, - name: - getErrorName(bucket.sample.hits.hits[0]._source) ?? - NOT_AVAILABLE_LABEL, - last_seen: new Date( - bucket.sample.hits.hits[0]?._source['@timestamp'] - ).getTime(), - occurrences: bucket.doc_count, - })) ?? []; + const errorGroups = + response.aggregations?.error_groups.buckets.map((bucket) => ({ + group_id: bucket.key as string, + name: + getErrorName(bucket.sample.hits.hits[0]._source) ?? NOT_AVAILABLE_LABEL, + last_seen: new Date( + bucket.sample.hits.hits[0]?._source['@timestamp'] + ).getTime(), + occurrences: bucket.doc_count, + })) ?? []; - return { - is_aggregation_accurate: - (response.aggregations?.error_groups.sum_other_doc_count ?? 0) === 0, - error_groups: errorGroups, - }; - }); + return { + is_aggregation_accurate: + (response.aggregations?.error_groups.sum_other_doc_count ?? 0) === 0, + error_groups: errorGroups, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts index 7729822df30ca0..b720c56464c30f 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_error_groups/index.ts @@ -59,8 +59,9 @@ export async function getServiceErrorGroups({ const { intervalString } = getBucketSize({ start, end, numBuckets }); - const response = await withApmSpan('get_top_service_error_groups', () => - apmEventClient.search({ + const response = await apmEventClient.search( + 'get_top_service_error_groups', + { apm: { events: [ProcessorEvent.error], }, @@ -104,7 +105,7 @@ export async function getServiceErrorGroups({ }, }, }, - }) + } ); const errorGroups = @@ -139,50 +140,49 @@ export async function getServiceErrorGroups({ (group) => group.group_id ); - const timeseriesResponse = await withApmSpan( + const timeseriesResponse = await apmEventClient.search( 'get_service_error_groups_timeseries', - async () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.error], + { + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { terms: { [ERROR_GROUP_ID]: sortedErrorGroupIds } }, + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + }, }, - body: { - size: 0, - query: { - bool: { - filter: [ - { terms: { [ERROR_GROUP_ID]: sortedErrorGroupIds } }, - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], + aggs: { + error_groups: { + terms: { + field: ERROR_GROUP_ID, + size, }, - }, - aggs: { - error_groups: { - terms: { - field: ERROR_GROUP_ID, - size, - }, - aggs: { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { - min: start, - max: end, - }, + aggs: { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, }, }, }, }, }, }, - }) + }, + } ); return { diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instance_metadata_details.ts b/x-pack/plugins/apm/server/lib/services/get_service_instance_metadata_details.ts index 25935bcc37dff5..bdf9530a9c0c7e 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instance_metadata_details.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instance_metadata_details.ts @@ -11,7 +11,6 @@ import { TRANSACTION_TYPE, } from '../../../common/elasticsearch_fieldnames'; import { environmentQuery, kqlQuery, rangeQuery } from '../../utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; @@ -37,18 +36,19 @@ export async function getServiceInstanceMetadataDetails({ environment?: string; kuery?: string; }) { - return withApmSpan('get_service_instance_metadata_details', async () => { - const { start, end, apmEventClient } = setup; - const filter = [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [SERVICE_NODE_NAME]: serviceNodeName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ]; + const { start, end, apmEventClient } = setup; + const filter = [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [SERVICE_NODE_NAME]: serviceNodeName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ]; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_instance_metadata_details', + { apm: { events: [ getProcessorEventForAggregatedTransactions( @@ -61,24 +61,24 @@ export async function getServiceInstanceMetadataDetails({ size: 1, query: { bool: { filter } }, }, - }); + } + ); - const sample = response.hits.hits[0]?._source; + const sample = response.hits.hits[0]?._source; - if (!sample) { - return {}; - } + if (!sample) { + return {}; + } - const { agent, service, container, kubernetes, host, cloud } = sample; + const { agent, service, container, kubernetes, host, cloud } = sample; - return { - '@timestamp': sample['@timestamp'], - agent, - service, - container, - kubernetes, - host, - cloud, - }; - }); + return { + '@timestamp': sample['@timestamp'], + agent, + service, + container, + kubernetes, + host, + cloud, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_system_metric_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_system_metric_statistics.ts index 1a33e9810dd5e0..526ae19143f130 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_system_metric_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_system_metric_statistics.ts @@ -24,7 +24,6 @@ import { percentCgroupMemoryUsedScript, percentSystemMemoryUsedScript, } from '../../metrics/by_agent/shared/memory'; -import { withApmSpan } from '../../../utils/with_apm_span'; interface ServiceInstanceSystemMetricPrimaryStatistics { serviceNodeName: string; @@ -67,142 +66,140 @@ export async function getServiceInstancesSystemMetricStatistics< size?: number; isComparisonSearch: T; }): Promise>> { - return withApmSpan( - 'get_service_instances_system_metric_statistics', - async () => { - const { apmEventClient } = setup; - - const { intervalString } = getBucketSize({ start, end, numBuckets }); - - const systemMemoryFilter = { - bool: { - filter: [ - { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, - { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, - ], - }, - }; - - const cgroupMemoryFilter = { - exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES }, - }; - - const cpuUsageFilter = { exists: { field: METRIC_PROCESS_CPU_PERCENT } }; - - function withTimeseries( - agg: TParams - ) { - return { - ...(isComparisonSearch - ? { - avg: { avg: agg }, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { - min: start, - max: end, - }, - }, - aggs: { avg: { avg: agg } }, + const { apmEventClient } = setup; + + const { intervalString } = getBucketSize({ start, end, numBuckets }); + + const systemMemoryFilter = { + bool: { + filter: [ + { exists: { field: METRIC_SYSTEM_FREE_MEMORY } }, + { exists: { field: METRIC_SYSTEM_TOTAL_MEMORY } }, + ], + }, + }; + + const cgroupMemoryFilter = { + exists: { field: METRIC_CGROUP_MEMORY_USAGE_BYTES }, + }; + + const cpuUsageFilter = { exists: { field: METRIC_PROCESS_CPU_PERCENT } }; + + function withTimeseries( + agg: TParams + ) { + return { + ...(isComparisonSearch + ? { + avg: { avg: agg }, + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, }, - } - : { avg: { avg: agg } }), - }; - } - - const subAggs = { - memory_usage_cgroup: { - filter: cgroupMemoryFilter, - aggs: withTimeseries({ script: percentCgroupMemoryUsedScript }), - }, - memory_usage_system: { - filter: systemMemoryFilter, - aggs: withTimeseries({ script: percentSystemMemoryUsedScript }), - }, - cpu_usage: { - filter: cpuUsageFilter, - aggs: withTimeseries({ field: METRIC_PROCESS_CPU_PERCENT }), - }, - }; - - const response = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.metric], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ...(isComparisonSearch && serviceNodeIds - ? [{ terms: { [SERVICE_NODE_NAME]: serviceNodeIds } }] - : []), - ], - should: [cgroupMemoryFilter, systemMemoryFilter, cpuUsageFilter], - minimum_should_match: 1, + }, + aggs: { avg: { avg: agg } }, }, + } + : { avg: { avg: agg } }), + }; + } + + const subAggs = { + memory_usage_cgroup: { + filter: cgroupMemoryFilter, + aggs: withTimeseries({ script: percentCgroupMemoryUsedScript }), + }, + memory_usage_system: { + filter: systemMemoryFilter, + aggs: withTimeseries({ script: percentSystemMemoryUsedScript }), + }, + cpu_usage: { + filter: cpuUsageFilter, + aggs: withTimeseries({ field: METRIC_PROCESS_CPU_PERCENT }), + }, + }; + + const response = await apmEventClient.search( + 'get_service_instances_system_metric_statistics', + { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...(isComparisonSearch && serviceNodeIds + ? [{ terms: { [SERVICE_NODE_NAME]: serviceNodeIds } }] + : []), + ], + should: [cgroupMemoryFilter, systemMemoryFilter, cpuUsageFilter], + minimum_should_match: 1, }, - aggs: { - [SERVICE_NODE_NAME]: { - terms: { - field: SERVICE_NODE_NAME, - missing: SERVICE_NODE_NAME_MISSING, - ...(size ? { size } : {}), - ...(isComparisonSearch ? { include: serviceNodeIds } : {}), - }, - aggs: subAggs, + }, + aggs: { + [SERVICE_NODE_NAME]: { + terms: { + field: SERVICE_NODE_NAME, + missing: SERVICE_NODE_NAME_MISSING, + ...(size ? { size } : {}), + ...(isComparisonSearch ? { include: serviceNodeIds } : {}), }, + aggs: subAggs, }, }, - }); - - return ( - (response.aggregations?.[SERVICE_NODE_NAME].buckets.map( - (serviceNodeBucket) => { - const serviceNodeName = String(serviceNodeBucket.key); - const hasCGroupData = - serviceNodeBucket.memory_usage_cgroup.avg.value !== null; - - const memoryMetricsKey = hasCGroupData - ? 'memory_usage_cgroup' - : 'memory_usage_system'; - - const cpuUsage = - // Timeseries is available when isComparisonSearch is true - 'timeseries' in serviceNodeBucket.cpu_usage - ? serviceNodeBucket.cpu_usage.timeseries.buckets.map( - (dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.avg.value, - }) - ) - : serviceNodeBucket.cpu_usage.avg.value; - - const memoryUsageValue = serviceNodeBucket[memoryMetricsKey]; - const memoryUsage = - // Timeseries is available when isComparisonSearch is true - 'timeseries' in memoryUsageValue - ? memoryUsageValue.timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.avg.value, - })) - : serviceNodeBucket[memoryMetricsKey].avg.value; - - return { - serviceNodeName, - cpuUsage, - memoryUsage, - }; - } - ) as Array>) || [] - ); + }, } ); + + return ( + (response.aggregations?.[SERVICE_NODE_NAME].buckets.map( + (serviceNodeBucket) => { + const serviceNodeName = String(serviceNodeBucket.key); + const hasCGroupData = + serviceNodeBucket.memory_usage_cgroup.avg.value !== null; + + const memoryMetricsKey = hasCGroupData + ? 'memory_usage_cgroup' + : 'memory_usage_system'; + + const cpuUsage = + // Timeseries is available when isComparisonSearch is true + 'timeseries' in serviceNodeBucket.cpu_usage + ? serviceNodeBucket.cpu_usage.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.avg.value, + }) + ) + : serviceNodeBucket.cpu_usage.avg.value; + + const memoryUsageValue = serviceNodeBucket[memoryMetricsKey]; + const memoryUsage = + // Timeseries is available when isComparisonSearch is true + 'timeseries' in memoryUsageValue + ? memoryUsageValue.timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.avg.value, + })) + : serviceNodeBucket[memoryMetricsKey].avg.value; + + return { + serviceNodeName, + cpuUsage, + memoryUsage, + }; + } + ) as Array>) || [] + ); } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts index ad54a231b52ef2..7d9dca9b2a7061 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_instances/get_service_instances_transaction_statistics.ts @@ -26,7 +26,6 @@ import { getLatencyValue, } from '../../helpers/latency_aggregation_type'; import { Setup } from '../../helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; interface ServiceInstanceTransactionPrimaryStatistics { serviceNodeName: string; @@ -77,126 +76,124 @@ export async function getServiceInstancesTransactionStatistics< size?: number; numBuckets?: number; }): Promise>> { - return withApmSpan( - 'get_service_instances_transaction_statistics', - async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const { intervalString, bucketSize } = getBucketSize({ - start, - end, - numBuckets, - }); + const { intervalString, bucketSize } = getBucketSize({ + start, + end, + numBuckets, + }); - const field = getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ); + const field = getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ); - const subAggs = { - ...getLatencyAggregation(latencyAggregationType, field), - failures: { - filter: { - term: { - [EVENT_OUTCOME]: EventOutcome.failure, - }, - }, + const subAggs = { + ...getLatencyAggregation(latencyAggregationType, field), + failures: { + filter: { + term: { + [EVENT_OUTCOME]: EventOutcome.failure, }, - }; + }, + }, + }; - const query = { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ...(isComparisonSearch && serviceNodeIds - ? [{ terms: { [SERVICE_NODE_NAME]: serviceNodeIds } }] - : []), - ], - }, - }; + const query = { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ...(isComparisonSearch && serviceNodeIds + ? [{ terms: { [SERVICE_NODE_NAME]: serviceNodeIds } }] + : []), + ], + }, + }; - const aggs = { - [SERVICE_NODE_NAME]: { - terms: { - field: SERVICE_NODE_NAME, - missing: SERVICE_NODE_NAME_MISSING, - ...(size ? { size } : {}), - ...(isComparisonSearch ? { include: serviceNodeIds } : {}), - }, - aggs: isComparisonSearch - ? { - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { min: start, max: end }, - }, - aggs: subAggs, - }, - } - : subAggs, - }, - }; + const aggs = { + [SERVICE_NODE_NAME]: { + terms: { + field: SERVICE_NODE_NAME, + missing: SERVICE_NODE_NAME_MISSING, + ...(size ? { size } : {}), + ...(isComparisonSearch ? { include: serviceNodeIds } : {}), + }, + aggs: isComparisonSearch + ? { + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { min: start, max: end }, + }, + aggs: subAggs, + }, + } + : subAggs, + }, + }; - const response = await apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { size: 0, query, aggs }, - }); + const response = await apmEventClient.search( + 'get_service_instances_transaction_statistics', + { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { size: 0, query, aggs }, + } + ); - const bucketSizeInMinutes = bucketSize / 60; + const bucketSizeInMinutes = bucketSize / 60; - return ( - (response.aggregations?.[SERVICE_NODE_NAME].buckets.map( - (serviceNodeBucket) => { - const { doc_count: count, key } = serviceNodeBucket; - const serviceNodeName = String(key); + return ( + (response.aggregations?.[SERVICE_NODE_NAME].buckets.map( + (serviceNodeBucket) => { + const { doc_count: count, key } = serviceNodeBucket; + const serviceNodeName = String(key); - // Timeseries is returned when isComparisonSearch is true - if ('timeseries' in serviceNodeBucket) { - const { timeseries } = serviceNodeBucket; - return { - serviceNodeName, - errorRate: timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.failures.doc_count / dateBucket.doc_count, - })), - throughput: timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.doc_count / bucketSizeInMinutes, - })), - latency: timeseries.buckets.map((dateBucket) => ({ - x: dateBucket.key, - y: getLatencyValue({ - aggregation: dateBucket.latency, - latencyAggregationType, - }), - })), - }; - } else { - const { failures, latency } = serviceNodeBucket; - return { - serviceNodeName, - errorRate: failures.doc_count / count, - latency: getLatencyValue({ - aggregation: latency, - latencyAggregationType, - }), - throughput: calculateThroughput({ start, end, value: count }), - }; - } - } - ) as Array>) || [] - ); - } + // Timeseries is returned when isComparisonSearch is true + if ('timeseries' in serviceNodeBucket) { + const { timeseries } = serviceNodeBucket; + return { + serviceNodeName, + errorRate: timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.failures.doc_count / dateBucket.doc_count, + })), + throughput: timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.doc_count / bucketSizeInMinutes, + })), + latency: timeseries.buckets.map((dateBucket) => ({ + x: dateBucket.key, + y: getLatencyValue({ + aggregation: dateBucket.latency, + latencyAggregationType, + }), + })), + }; + } else { + const { failures, latency } = serviceNodeBucket; + return { + serviceNodeName, + errorRate: failures.doc_count / count, + latency: getLatencyValue({ + aggregation: latency, + latencyAggregationType, + }), + throughput: calculateThroughput({ start, end, value: count }), + }; + } + } + ) as Array>) || [] ); } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_metadata_details.ts b/x-pack/plugins/apm/server/lib/services/get_service_metadata_details.ts index e2341b306a8781..910725b0054113 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_metadata_details.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_metadata_details.ts @@ -25,7 +25,6 @@ import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw' import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { should } from './get_service_metadata_icons'; -import { withApmSpan } from '../../utils/with_apm_span'; type ServiceMetadataDetailsRaw = Pick< TransactionRaw, @@ -59,7 +58,7 @@ export interface ServiceMetadataDetails { }; } -export function getServiceMetadataDetails({ +export async function getServiceMetadataDetails({ serviceName, setup, searchAggregatedTransactions, @@ -68,105 +67,106 @@ export function getServiceMetadataDetails({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }): Promise { - return withApmSpan('get_service_metadata_details', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const filter = [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ]; + const filter = [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ]; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - size: 1, - _source: [SERVICE, AGENT, HOST, CONTAINER_ID, KUBERNETES, CLOUD], - query: { bool: { filter, should } }, - aggs: { - serviceVersions: { - terms: { - field: SERVICE_VERSION, - size: 10, - order: { _key: 'desc' as const }, - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + size: 1, + _source: [SERVICE, AGENT, HOST, CONTAINER_ID, KUBERNETES, CLOUD], + query: { bool: { filter, should } }, + aggs: { + serviceVersions: { + terms: { + field: SERVICE_VERSION, + size: 10, + order: { _key: 'desc' as const }, }, - availabilityZones: { - terms: { - field: CLOUD_AVAILABILITY_ZONE, - size: 10, - }, + }, + availabilityZones: { + terms: { + field: CLOUD_AVAILABILITY_ZONE, + size: 10, }, - machineTypes: { - terms: { - field: CLOUD_MACHINE_TYPE, - size: 10, - }, + }, + machineTypes: { + terms: { + field: CLOUD_MACHINE_TYPE, + size: 10, }, - totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, + totalNumberInstances: { cardinality: { field: SERVICE_NODE_NAME } }, }, - }; - - const response = await apmEventClient.search(params); - - if (response.hits.total.value === 0) { - return { - service: undefined, - container: undefined, - cloud: undefined, - }; - } + }, + }; - const { service, agent, host, kubernetes, container, cloud } = response.hits - .hits[0]._source as ServiceMetadataDetailsRaw; + const response = await apmEventClient.search( + 'get_service_metadata_details', + params + ); - const serviceMetadataDetails = { - versions: response.aggregations?.serviceVersions.buckets.map( - (bucket) => bucket.key as string - ), - runtime: service.runtime, - framework: service.framework?.name, - agent, + if (response.hits.total.value === 0) { + return { + service: undefined, + container: undefined, + cloud: undefined, }; + } + + const { service, agent, host, kubernetes, container, cloud } = response.hits + .hits[0]._source as ServiceMetadataDetailsRaw; - const totalNumberInstances = - response.aggregations?.totalNumberInstances.value; + const serviceMetadataDetails = { + versions: response.aggregations?.serviceVersions.buckets.map( + (bucket) => bucket.key as string + ), + runtime: service.runtime, + framework: service.framework?.name, + agent, + }; - const containerDetails = - host || container || totalNumberInstances || kubernetes - ? { - os: host?.os?.platform, - type: (!!kubernetes ? 'Kubernetes' : 'Docker') as ContainerType, - isContainerized: !!container?.id, - totalNumberInstances, - } - : undefined; + const totalNumberInstances = + response.aggregations?.totalNumberInstances.value; - const cloudDetails = cloud + const containerDetails = + host || container || totalNumberInstances || kubernetes ? { - provider: cloud.provider, - projectName: cloud.project?.name, - availabilityZones: response.aggregations?.availabilityZones.buckets.map( - (bucket) => bucket.key as string - ), - machineTypes: response.aggregations?.machineTypes.buckets.map( - (bucket) => bucket.key as string - ), + os: host?.os?.platform, + type: (!!kubernetes ? 'Kubernetes' : 'Docker') as ContainerType, + isContainerized: !!container?.id, + totalNumberInstances, } : undefined; - return { - service: serviceMetadataDetails, - container: containerDetails, - cloud: cloudDetails, - }; - }); + const cloudDetails = cloud + ? { + provider: cloud.provider, + projectName: cloud.project?.name, + availabilityZones: response.aggregations?.availabilityZones.buckets.map( + (bucket) => bucket.key as string + ), + machineTypes: response.aggregations?.machineTypes.buckets.map( + (bucket) => bucket.key as string + ), + } + : undefined; + + return { + service: serviceMetadataDetails, + container: containerDetails, + cloud: cloudDetails, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_metadata_icons.ts b/x-pack/plugins/apm/server/lib/services/get_service_metadata_icons.ts index 94da6545c5e90e..469c788a6cf17a 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_metadata_icons.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_metadata_icons.ts @@ -20,7 +20,6 @@ import { rangeQuery } from '../../../server/utils/queries'; import { TransactionRaw } from '../../../typings/es_schemas/raw/transaction_raw'; import { getProcessorEventForAggregatedTransactions } from '../helpers/aggregated_transactions'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; -import { withApmSpan } from '../../utils/with_apm_span'; type ServiceMetadataIconsRaw = Pick< TransactionRaw, @@ -41,7 +40,7 @@ export const should = [ { exists: { field: AGENT_NAME } }, ]; -export function getServiceMetadataIcons({ +export async function getServiceMetadataIcons({ serviceName, setup, searchAggregatedTransactions, @@ -50,55 +49,56 @@ export function getServiceMetadataIcons({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }): Promise { - return withApmSpan('get_service_metadata_icons', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const filter = [ - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ]; + const filter = [ + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ]; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - size: 1, - _source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME], - query: { bool: { filter, should } }, - }, - }; + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + size: 1, + _source: [KUBERNETES, CLOUD_PROVIDER, CONTAINER_ID, AGENT_NAME], + query: { bool: { filter, should } }, + }, + }; - const response = await apmEventClient.search(params); + const response = await apmEventClient.search( + 'get_service_metadata_icons', + params + ); - if (response.hits.total.value === 0) { - return { - agentName: undefined, - containerType: undefined, - cloudProvider: undefined, - }; - } + if (response.hits.total.value === 0) { + return { + agentName: undefined, + containerType: undefined, + cloudProvider: undefined, + }; + } - const { kubernetes, cloud, container, agent } = response.hits.hits[0] - ._source as ServiceMetadataIconsRaw; + const { kubernetes, cloud, container, agent } = response.hits.hits[0] + ._source as ServiceMetadataIconsRaw; - let containerType: ContainerType; - if (!!kubernetes) { - containerType = 'Kubernetes'; - } else if (!!container) { - containerType = 'Docker'; - } + let containerType: ContainerType; + if (!!kubernetes) { + containerType = 'Kubernetes'; + } else if (!!container) { + containerType = 'Docker'; + } - return { - agentName: agent?.name, - containerType, - cloudProvider: cloud?.provider, - }; - }); + return { + agentName: agent?.name, + containerType, + cloudProvider: cloud?.provider, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_node_metadata.ts b/x-pack/plugins/apm/server/lib/services/get_service_node_metadata.ts index 8eaf9e96c7fd97..0f0c1741790526 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_node_metadata.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_node_metadata.ts @@ -13,9 +13,8 @@ import { import { NOT_AVAILABLE_LABEL } from '../../../common/i18n'; import { mergeProjection } from '../../projections/util/merge_projection'; import { getServiceNodesProjection } from '../../projections/service_nodes'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getServiceNodeMetadata({ +export async function getServiceNodeMetadata({ kuery, serviceName, serviceNodeName, @@ -26,44 +25,44 @@ export function getServiceNodeMetadata({ serviceNodeName: string; setup: Setup & SetupTimeRange; }) { - return withApmSpan('get_service_node_metadata', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const query = mergeProjection( - getServiceNodesProjection({ - kuery, - setup, - serviceName, - serviceNodeName, - }), - { - body: { - size: 0, - aggs: { - host: { - terms: { - field: HOST_NAME, - size: 1, - }, + const query = mergeProjection( + getServiceNodesProjection({ + kuery, + setup, + serviceName, + serviceNodeName, + }), + { + body: { + size: 0, + aggs: { + host: { + terms: { + field: HOST_NAME, + size: 1, }, - containerId: { - terms: { - field: CONTAINER_ID, - size: 1, - }, + }, + containerId: { + terms: { + field: CONTAINER_ID, + size: 1, }, }, }, - } - ); + }, + } + ); - const response = await apmEventClient.search(query); + const response = await apmEventClient.search( + 'get_service_node_metadata', + query + ); - return { - host: response.aggregations?.host.buckets[0]?.key || NOT_AVAILABLE_LABEL, - containerId: - response.aggregations?.containerId.buckets[0]?.key || - NOT_AVAILABLE_LABEL, - }; - }); + return { + host: response.aggregations?.host.buckets[0]?.key || NOT_AVAILABLE_LABEL, + containerId: + response.aggregations?.containerId.buckets[0]?.key || NOT_AVAILABLE_LABEL, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts index f14dba69bf404b..36d372e322cbc4 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_group_detailed_statistics.ts @@ -21,7 +21,6 @@ import { kqlQuery, } from '../../../server/utils/queries'; import { Coordinate } from '../../../typings/timeseries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, @@ -68,74 +67,72 @@ export async function getServiceTransactionGroupDetailedStatistics({ impact: number; }> > { - return withApmSpan( - 'get_service_transaction_group_detailed_statistics', - async () => { - const { apmEventClient } = setup; - const { intervalString } = getBucketSize({ start, end, numBuckets }); + const { apmEventClient } = setup; + const { intervalString } = getBucketSize({ start, end, numBuckets }); - const field = getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ); + const field = getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ); - const response = await apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], + const response = await apmEventClient.search( + 'get_service_transaction_group_detailed_statistics', + { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], + }, }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], + aggs: { + total_duration: { sum: { field } }, + transaction_groups: { + terms: { + field: TRANSACTION_NAME, + include: transactionNames, + size: transactionNames.length, }, - }, - aggs: { - total_duration: { sum: { field } }, - transaction_groups: { - terms: { - field: TRANSACTION_NAME, - include: transactionNames, - size: transactionNames.length, + aggs: { + transaction_group_total_duration: { + sum: { field }, }, - aggs: { - transaction_group_total_duration: { - sum: { field }, + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: intervalString, + min_doc_count: 0, + extended_bounds: { + min: start, + max: end, + }, }, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: intervalString, - min_doc_count: 0, - extended_bounds: { - min: start, - max: end, + aggs: { + throughput_rate: { + rate: { + unit: 'minute', }, }, - aggs: { - throughput_rate: { - rate: { - unit: 'minute', - }, - }, - ...getLatencyAggregation(latencyAggregationType, field), - [EVENT_OUTCOME]: { - terms: { - field: EVENT_OUTCOME, - include: [EventOutcome.failure, EventOutcome.success], - }, + ...getLatencyAggregation(latencyAggregationType, field), + [EVENT_OUTCOME]: { + terms: { + field: EVENT_OUTCOME, + include: [EventOutcome.failure, EventOutcome.success], }, }, }, @@ -143,46 +140,42 @@ export async function getServiceTransactionGroupDetailedStatistics({ }, }, }, - }); - - const buckets = response.aggregations?.transaction_groups.buckets ?? []; - - const totalDuration = response.aggregations?.total_duration.value; - return buckets.map((bucket) => { - const transactionName = bucket.key as string; - const latency = bucket.timeseries.buckets.map((timeseriesBucket) => ({ - x: timeseriesBucket.key, - y: getLatencyValue({ - latencyAggregationType, - aggregation: timeseriesBucket.latency, - }), - })); - const throughput = bucket.timeseries.buckets.map( - (timeseriesBucket) => ({ - x: timeseriesBucket.key, - y: timeseriesBucket.throughput_rate.value, - }) - ); - const errorRate = bucket.timeseries.buckets.map((timeseriesBucket) => ({ - x: timeseriesBucket.key, - y: calculateTransactionErrorPercentage( - timeseriesBucket[EVENT_OUTCOME] - ), - })); - const transactionGroupTotalDuration = - bucket.transaction_group_total_duration.value || 0; - return { - transactionName, - latency, - throughput, - errorRate, - impact: totalDuration - ? (transactionGroupTotalDuration * 100) / totalDuration - : 0, - }; - }); + }, } ); + + const buckets = response.aggregations?.transaction_groups.buckets ?? []; + + const totalDuration = response.aggregations?.total_duration.value; + return buckets.map((bucket) => { + const transactionName = bucket.key as string; + const latency = bucket.timeseries.buckets.map((timeseriesBucket) => ({ + x: timeseriesBucket.key, + y: getLatencyValue({ + latencyAggregationType, + aggregation: timeseriesBucket.latency, + }), + })); + const throughput = bucket.timeseries.buckets.map((timeseriesBucket) => ({ + x: timeseriesBucket.key, + y: timeseriesBucket.throughput_rate.value, + })); + const errorRate = bucket.timeseries.buckets.map((timeseriesBucket) => ({ + x: timeseriesBucket.key, + y: calculateTransactionErrorPercentage(timeseriesBucket[EVENT_OUTCOME]), + })); + const transactionGroupTotalDuration = + bucket.transaction_group_total_duration.value || 0; + return { + transactionName, + latency, + throughput, + errorRate, + impact: totalDuration + ? (transactionGroupTotalDuration * 100) / totalDuration + : 0, + }; + }); } export async function getServiceTransactionGroupDetailedStatisticsPeriods({ diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups.ts index 28574bab4df213..a4cc27c875d731 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_groups.ts @@ -18,7 +18,6 @@ import { rangeQuery, kqlQuery, } from '../../../server/utils/queries'; -import { withApmSpan } from '../../utils/with_apm_span'; import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, @@ -56,14 +55,15 @@ export async function getServiceTransactionGroups({ transactionType: string; latencyAggregationType: LatencyAggregationType; }) { - return withApmSpan('get_service_transaction_groups', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const field = getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ); + const field = getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ); - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_transaction_groups', + { apm: { events: [ getProcessorEventForAggregatedTransactions( @@ -110,45 +110,45 @@ export async function getServiceTransactionGroups({ }, }, }, - }); + } + ); - const totalDuration = response.aggregations?.total_duration.value; + const totalDuration = response.aggregations?.total_duration.value; - const transactionGroups = - response.aggregations?.transaction_groups.buckets.map((bucket) => { - const errorRate = calculateTransactionErrorPercentage( - bucket[EVENT_OUTCOME] - ); + const transactionGroups = + response.aggregations?.transaction_groups.buckets.map((bucket) => { + const errorRate = calculateTransactionErrorPercentage( + bucket[EVENT_OUTCOME] + ); - const transactionGroupTotalDuration = - bucket.transaction_group_total_duration.value || 0; + const transactionGroupTotalDuration = + bucket.transaction_group_total_duration.value || 0; - return { - name: bucket.key as string, - latency: getLatencyValue({ - latencyAggregationType, - aggregation: bucket.latency, - }), - throughput: calculateThroughput({ - start, - end, - value: bucket.doc_count, - }), - errorRate, - impact: totalDuration - ? (transactionGroupTotalDuration * 100) / totalDuration - : 0, - }; - }) ?? []; + return { + name: bucket.key as string, + latency: getLatencyValue({ + latencyAggregationType, + aggregation: bucket.latency, + }), + throughput: calculateThroughput({ + start, + end, + value: bucket.doc_count, + }), + errorRate, + impact: totalDuration + ? (transactionGroupTotalDuration * 100) / totalDuration + : 0, + }; + }) ?? []; - return { - transactionGroups: transactionGroups.map((transactionGroup) => ({ - ...transactionGroup, - transactionType, - })), - isAggregationAccurate: - (response.aggregations?.transaction_groups.sum_other_doc_count ?? 0) === - 0, - }; - }); + return { + transactionGroups: transactionGroups.map((transactionGroup) => ({ + ...transactionGroup, + transactionType, + })), + isAggregationAccurate: + (response.aggregations?.transaction_groups.sum_other_doc_count ?? 0) === + 0, + }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_service_transaction_types.ts b/x-pack/plugins/apm/server/lib/services/get_service_transaction_types.ts index e280ab6db1665d..f38a7fba09d967 100644 --- a/x-pack/plugins/apm/server/lib/services/get_service_transaction_types.ts +++ b/x-pack/plugins/apm/server/lib/services/get_service_transaction_types.ts @@ -15,9 +15,8 @@ import { getDocumentTypeFilterForAggregatedTransactions, getProcessorEventForAggregatedTransactions, } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; -export function getServiceTransactionTypes({ +export async function getServiceTransactionTypes({ setup, serviceName, searchAggregatedTransactions, @@ -26,41 +25,42 @@ export function getServiceTransactionTypes({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_service_transaction_types', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - { term: { [SERVICE_NAME]: serviceName } }, - ...rangeQuery(start, end), - ], - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + { term: { [SERVICE_NAME]: serviceName } }, + ...rangeQuery(start, end), + ], }, - aggs: { - types: { - terms: { field: TRANSACTION_TYPE, size: 100 }, - }, + }, + aggs: { + types: { + terms: { field: TRANSACTION_TYPE, size: 100 }, }, }, - }; + }, + }; - const { aggregations } = await apmEventClient.search(params); - const transactionTypes = - aggregations?.types.buckets.map((bucket) => bucket.key as string) || []; - return { transactionTypes }; - }); + const { aggregations } = await apmEventClient.search( + 'get_service_transaction_types', + params + ); + const transactionTypes = + aggregations?.types.buckets.map((bucket) => bucket.key as string) || []; + return { transactionTypes }; } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_legacy_data_status.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_legacy_data_status.ts index b42fd340bfb422..f33bedb6ef4fb1 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_legacy_data_status.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_legacy_data_status.ts @@ -9,35 +9,31 @@ import { rangeQuery } from '../../../../server/utils/queries'; import { ProcessorEvent } from '../../../../common/processor_event'; import { OBSERVER_VERSION_MAJOR } from '../../../../common/elasticsearch_fieldnames'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; // returns true if 6.x data is found export async function getLegacyDataStatus(setup: Setup & SetupTimeRange) { - return withApmSpan('get_legacy_data_status', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const params = { - terminateAfter: 1, - apm: { - events: [ProcessorEvent.transaction], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { range: { [OBSERVER_VERSION_MAJOR]: { lt: 7 } } }, - ...rangeQuery(start, end), - ], - }, + const params = { + terminateAfter: 1, + apm: { + events: [ProcessorEvent.transaction], + includeLegacyData: true, + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { range: { [OBSERVER_VERSION_MAJOR]: { lt: 7 } } }, + ...rangeQuery(start, end), + ], }, }, - }; + }, + }; - const resp = await apmEventClient.search(params, { - includeLegacyData: true, - }); - const hasLegacyData = resp.hits.total.value > 0; - return hasLegacyData; - }); + const resp = await apmEventClient.search('get_legacy_data_status', params); + const hasLegacyData = resp.hits.total.value > 0; + return hasLegacyData; } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts index 1e36df379e9646..019ab8770887ae 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_service_transaction_stats.ts @@ -33,7 +33,6 @@ import { getOutcomeAggregation, } from '../../helpers/transaction_error_rate'; import { ServicesItemsSetup } from './get_services_items'; -import { withApmSpan } from '../../../utils/with_apm_span'; interface AggregationParams { environment?: string; @@ -50,23 +49,24 @@ export async function getServiceTransactionStats({ searchAggregatedTransactions, maxNumServices, }: AggregationParams) { - return withApmSpan('get_service_transaction_stats', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const outcomes = getOutcomeAggregation(); + const outcomes = getOutcomeAggregation(); - const metrics = { - avg_duration: { - avg: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, + const metrics = { + avg_duration: { + avg: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, - outcomes, - }; + }, + outcomes, + }; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_transaction_stats', + { apm: { events: [ getProcessorEventForAggregatedTransactions( @@ -133,64 +133,64 @@ export async function getServiceTransactionStats({ }, }, }, - }); + } + ); - return ( - response.aggregations?.services.buckets.map((bucket) => { - const topTransactionTypeBucket = - bucket.transactionType.buckets.find( - ({ key }) => - key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD - ) ?? bucket.transactionType.buckets[0]; + return ( + response.aggregations?.services.buckets.map((bucket) => { + const topTransactionTypeBucket = + bucket.transactionType.buckets.find( + ({ key }) => + key === TRANSACTION_REQUEST || key === TRANSACTION_PAGE_LOAD + ) ?? bucket.transactionType.buckets[0]; - return { - serviceName: bucket.key as string, - transactionType: topTransactionTypeBucket.key as string, - environments: topTransactionTypeBucket.environments.buckets.map( - (environmentBucket) => environmentBucket.key as string + return { + serviceName: bucket.key as string, + transactionType: topTransactionTypeBucket.key as string, + environments: topTransactionTypeBucket.environments.buckets.map( + (environmentBucket) => environmentBucket.key as string + ), + agentName: topTransactionTypeBucket.sample.top[0].metrics[ + AGENT_NAME + ] as AgentName, + avgResponseTime: { + value: topTransactionTypeBucket.avg_duration.value, + timeseries: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key, + y: dateBucket.avg_duration.value, + }) ), - agentName: topTransactionTypeBucket.sample.top[0].metrics[ - AGENT_NAME - ] as AgentName, - avgResponseTime: { - value: topTransactionTypeBucket.avg_duration.value, - timeseries: topTransactionTypeBucket.timeseries.buckets.map( - (dateBucket) => ({ - x: dateBucket.key, - y: dateBucket.avg_duration.value, - }) - ), - }, - transactionErrorRate: { - value: calculateTransactionErrorPercentage( - topTransactionTypeBucket.outcomes - ), - timeseries: topTransactionTypeBucket.timeseries.buckets.map( - (dateBucket) => ({ - x: dateBucket.key, - y: calculateTransactionErrorPercentage(dateBucket.outcomes), - }) - ), - }, - transactionsPerMinute: { - value: calculateThroughput({ - start, - end, - value: topTransactionTypeBucket.doc_count, - }), - timeseries: topTransactionTypeBucket.timeseries.buckets.map( - (dateBucket) => ({ - x: dateBucket.key, - y: calculateThroughput({ - start, - end, - value: dateBucket.doc_count, - }), - }) - ), - }, - }; - }) ?? [] - ); - }); + }, + transactionErrorRate: { + value: calculateTransactionErrorPercentage( + topTransactionTypeBucket.outcomes + ), + timeseries: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key, + y: calculateTransactionErrorPercentage(dateBucket.outcomes), + }) + ), + }, + transactionsPerMinute: { + value: calculateThroughput({ + start, + end, + value: topTransactionTypeBucket.doc_count, + }), + timeseries: topTransactionTypeBucket.timeseries.buckets.map( + (dateBucket) => ({ + x: dateBucket.key, + y: calculateThroughput({ + start, + end, + value: dateBucket.doc_count, + }), + }) + ), + }, + }; + }) ?? [] + ); } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts index 906cc62e64d1a4..4692d1122b16c9 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_from_metric_documents.ts @@ -14,9 +14,8 @@ import { import { environmentQuery, kqlQuery, rangeQuery } from '../../../utils/queries'; import { ProcessorEvent } from '../../../../common/processor_event'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getServicesFromMetricDocuments({ +export async function getServicesFromMetricDocuments({ environment, setup, maxNumServices, @@ -27,10 +26,11 @@ export function getServicesFromMetricDocuments({ maxNumServices: number; kuery?: string; }) { - return withApmSpan('get_services_from_metric_documents', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_services_from_metric_documents', + { apm: { events: [ProcessorEvent.metric], }, @@ -67,18 +67,18 @@ export function getServicesFromMetricDocuments({ }, }, }, - }); + } + ); - return ( - response.aggregations?.services.buckets.map((bucket) => { - return { - serviceName: bucket.key as string, - environments: bucket.environments.buckets.map( - (envBucket) => envBucket.key as string - ), - agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName, - }; - }) ?? [] - ); - }); + return ( + response.aggregations?.services.buckets.map((bucket) => { + return { + serviceName: bucket.key as string, + environments: bucket.environments.buckets.map( + (envBucket) => envBucket.key as string + ), + agentName: bucket.latest.top[0].metrics[AGENT_NAME] as AgentName, + }; + }) ?? [] + ); } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts b/x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts index 28f6944fd24daf..97b8a8fa5505b0 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/has_historical_agent_data.ts @@ -6,29 +6,26 @@ */ import { ProcessorEvent } from '../../../../common/processor_event'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; // Note: this logic is duplicated in tutorials/apm/envs/on_prem export async function hasHistoricalAgentData(setup: Setup) { - return withApmSpan('has_historical_agent_data', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const params = { - terminateAfter: 1, - apm: { - events: [ - ProcessorEvent.error, - ProcessorEvent.metric, - ProcessorEvent.transaction, - ], - }, - body: { - size: 0, - }, - }; + const params = { + terminateAfter: 1, + apm: { + events: [ + ProcessorEvent.error, + ProcessorEvent.metric, + ProcessorEvent.transaction, + ], + }, + body: { + size: 0, + }, + }; - const resp = await apmEventClient.search(params); - return resp.hits.total.value > 0; - }); + const resp = await apmEventClient.search('has_historical_agent_data', params); + return resp.hits.total.value > 0; } diff --git a/x-pack/plugins/apm/server/lib/services/get_throughput.ts b/x-pack/plugins/apm/server/lib/services/get_throughput.ts index 5f5008a28c2325..b0cb917d302fc5 100644 --- a/x-pack/plugins/apm/server/lib/services/get_throughput.ts +++ b/x-pack/plugins/apm/server/lib/services/get_throughput.ts @@ -21,7 +21,6 @@ import { } from '../helpers/aggregated_transactions'; import { getBucketSize } from '../helpers/get_bucket_size'; import { Setup } from '../helpers/setup_request'; -import { withApmSpan } from '../../utils/with_apm_span'; interface Options { environment?: string; @@ -88,20 +87,18 @@ function fetcher({ }, }; - return apmEventClient.search(params); + return apmEventClient.search('get_throughput_for_service', params); } -export function getThroughput(options: Options) { - return withApmSpan('get_throughput_for_service', async () => { - const response = await fetcher(options); +export async function getThroughput(options: Options) { + const response = await fetcher(options); - return ( - response.aggregations?.timeseries.buckets.map((bucket) => { - return { - x: bucket.key, - y: bucket.throughput.value, - }; - }) ?? [] - ); - }); + return ( + response.aggregations?.timeseries.buckets.map((bucket) => { + return { + x: bucket.key, + y: bucket.throughput.value, + }; + }) ?? [] + ); } diff --git a/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_statistics.ts b/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_statistics.ts index 858f36e6e2c13e..bb98abf724db4c 100644 --- a/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_statistics.ts +++ b/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_statistics.ts @@ -41,7 +41,7 @@ const maybeAdd = (to: any[], value: any) => { to.push(value); }; -function getProfilingStats({ +async function getProfilingStats({ apmEventClient, filter, valueTypeField, @@ -50,49 +50,47 @@ function getProfilingStats({ filter: ESFilter[]; valueTypeField: string; }) { - return withApmSpan('get_profiling_stats', async () => { - const response = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.profile], - }, - body: { - size: 0, - query: { - bool: { - filter, - }, + const response = await apmEventClient.search('get_profiling_stats', { + apm: { + events: [ProcessorEvent.profile], + }, + body: { + size: 0, + query: { + bool: { + filter, }, - aggs: { - stacks: { - terms: { - field: PROFILE_TOP_ID, - size: MAX_STACK_IDS, - order: { - value: 'desc', - }, + }, + aggs: { + stacks: { + terms: { + field: PROFILE_TOP_ID, + size: MAX_STACK_IDS, + order: { + value: 'desc', }, - aggs: { - value: { - sum: { - field: valueTypeField, - }, + }, + aggs: { + value: { + sum: { + field: valueTypeField, }, }, }, }, }, - }); + }, + }); - const stacks = - response.aggregations?.stacks.buckets.map((stack) => { - return { - id: stack.key as string, - value: stack.value.value!, - }; - }) ?? []; + const stacks = + response.aggregations?.stacks.buckets.map((stack) => { + return { + id: stack.key as string, + value: stack.value.value!, + }; + }) ?? []; - return stacks; - }); + return stacks; } function getProfilesWithStacks({ @@ -103,8 +101,9 @@ function getProfilesWithStacks({ filter: ESFilter[]; }) { return withApmSpan('get_profiles_with_stacks', async () => { - const cardinalityResponse = await withApmSpan('get_top_cardinality', () => - apmEventClient.search({ + const cardinalityResponse = await apmEventClient.search( + 'get_top_cardinality', + { apm: { events: [ProcessorEvent.profile], }, @@ -121,7 +120,7 @@ function getProfilesWithStacks({ }, }, }, - }) + } ); const cardinality = cardinalityResponse.aggregations?.top.value ?? 0; @@ -140,39 +139,37 @@ function getProfilesWithStacks({ const allResponses = await withApmSpan('get_all_stacks', async () => { return Promise.all( [...new Array(partitions)].map(async (_, num) => { - const response = await withApmSpan('get_partition', () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.profile], - }, - body: { - query: { - bool: { - filter, - }, + const response = await apmEventClient.search('get_partition', { + apm: { + events: [ProcessorEvent.profile], + }, + body: { + query: { + bool: { + filter, }, - aggs: { - top: { - terms: { - field: PROFILE_TOP_ID, - size: Math.max(MAX_STACKS_PER_REQUEST), - include: { - num_partitions: partitions, - partition: num, - }, + }, + aggs: { + top: { + terms: { + field: PROFILE_TOP_ID, + size: Math.max(MAX_STACKS_PER_REQUEST), + include: { + num_partitions: partitions, + partition: num, }, - aggs: { - latest: { - top_hits: { - _source: [PROFILE_TOP_ID, PROFILE_STACK], - }, + }, + aggs: { + latest: { + top_hits: { + _source: [PROFILE_TOP_ID, PROFILE_STACK], }, }, }, }, }, - }) - ); + }, + }); return ( response.aggregations?.top.buckets.flatMap((bucket) => { diff --git a/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_timeline.ts b/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_timeline.ts index 93fa029da8c721..af3cd6596a8c1b 100644 --- a/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_timeline.ts +++ b/x-pack/plugins/apm/server/lib/services/profiling/get_service_profiling_timeline.ts @@ -17,7 +17,6 @@ import { } from '../../../../common/profiling'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { getBucketSize } from '../../helpers/get_bucket_size'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { kqlQuery } from '../../../utils/queries'; const configMap = mapValues( @@ -38,10 +37,11 @@ export async function getServiceProfilingTimeline({ setup: Setup & SetupTimeRange; environment?: string; }) { - return withApmSpan('get_service_profiling_timeline', async () => { - const { apmEventClient, start, end } = setup; + const { apmEventClient, start, end } = setup; - const response = await apmEventClient.search({ + const response = await apmEventClient.search( + 'get_service_profiling_timeline', + { apm: { events: [ProcessorEvent.profile], }, @@ -96,29 +96,29 @@ export async function getServiceProfilingTimeline({ }, }, }, - }); + } + ); - const { aggregations } = response; + const { aggregations } = response; - if (!aggregations) { - return []; - } + if (!aggregations) { + return []; + } - return aggregations.timeseries.buckets.map((bucket) => { - return { - x: bucket.key, - valueTypes: { - unknown: bucket.value_type.buckets.unknown.num_profiles.value, - // TODO: use enum as object key. not possible right now - // because of https://github.com/microsoft/TypeScript/issues/37888 - ...mapValues(configMap, (_, key) => { - return ( - bucket.value_type.buckets[key as ProfilingValueType]?.num_profiles - .value ?? 0 - ); - }), - }, - }; - }); + return aggregations.timeseries.buckets.map((bucket) => { + return { + x: bucket.key, + valueTypes: { + unknown: bucket.value_type.buckets.unknown.num_profiles.value, + // TODO: use enum as object key. not possible right now + // because of https://github.com/microsoft/TypeScript/issues/37888 + ...mapValues(configMap, (_, key) => { + return ( + bucket.value_type.buckets[key as ProfilingValueType]?.num_profiles + .value ?? 0 + ); + }), + }, + }; }); } diff --git a/x-pack/plugins/apm/server/lib/services/queries.test.ts b/x-pack/plugins/apm/server/lib/services/queries.test.ts index b167eff65ee0aa..6adaca9c1a93dc 100644 --- a/x-pack/plugins/apm/server/lib/services/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/services/queries.test.ts @@ -55,7 +55,7 @@ describe('services queries', () => { }) ); - const allParams = mock.spy.mock.calls.map((call) => call[0]); + const allParams = mock.spy.mock.calls.map((call) => call[1]); expect(allParams).toMatchSnapshot(); }); diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_or_update_configuration.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_or_update_configuration.ts index 18853824355622..c112c3be3362b3 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_or_update_configuration.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/create_or_update_configuration.ts @@ -12,7 +12,6 @@ import { AgentConfigurationIntake, } from '../../../../common/agent_configuration/configuration_types'; import { APMIndexDocumentParams } from '../../helpers/create_es_client/create_internal_es_client'; -import { withApmSpan } from '../../../utils/with_apm_span'; export function createOrUpdateConfiguration({ configurationId, @@ -23,30 +22,28 @@ export function createOrUpdateConfiguration({ configurationIntake: AgentConfigurationIntake; setup: Setup; }) { - return withApmSpan('create_or_update_configuration', async () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - const params: APMIndexDocumentParams = { - refresh: true, - index: indices.apmAgentConfigurationIndex, - body: { - agent_name: configurationIntake.agent_name, - service: { - name: configurationIntake.service.name, - environment: configurationIntake.service.environment, - }, - settings: configurationIntake.settings, - '@timestamp': Date.now(), - applied_by_agent: false, - etag: hash(configurationIntake), + const params: APMIndexDocumentParams = { + refresh: true, + index: indices.apmAgentConfigurationIndex, + body: { + agent_name: configurationIntake.agent_name, + service: { + name: configurationIntake.service.name, + environment: configurationIntake.service.environment, }, - }; + settings: configurationIntake.settings, + '@timestamp': Date.now(), + applied_by_agent: false, + etag: hash(configurationIntake), + }, + }; - // by specifying an id elasticsearch will delete the previous doc and insert the updated doc - if (configurationId) { - params.id = configurationId; - } + // by specifying an id elasticsearch will delete the previous doc and insert the updated doc + if (configurationId) { + params.id = configurationId; + } - return internalClient.index(params); - }); + return internalClient.index('create_or_update_agent_configuration', params); } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts index 6ed6f79979889b..125c97730a6fa1 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/delete_configuration.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; export async function deleteConfiguration({ @@ -15,15 +14,13 @@ export async function deleteConfiguration({ configurationId: string; setup: Setup; }) { - return withApmSpan('delete_agent_configuration', async () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - const params = { - refresh: 'wait_for' as const, - index: indices.apmAgentConfigurationIndex, - id: configurationId, - }; + const params = { + refresh: 'wait_for' as const, + index: indices.apmAgentConfigurationIndex, + id: configurationId, + }; - return internalClient.delete(params); - }); + return internalClient.delete('delete_agent_configuration', params); } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/find_exact_configuration.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/find_exact_configuration.ts index 9fd4849c7640a8..3543d38f7b5d1c 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/find_exact_configuration.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/find_exact_configuration.ts @@ -11,47 +11,45 @@ import { SERVICE_ENVIRONMENT, SERVICE_NAME, } from '../../../../common/elasticsearch_fieldnames'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; import { convertConfigSettingsToString } from './convert_settings_to_string'; -export function findExactConfiguration({ +export async function findExactConfiguration({ service, setup, }: { service: AgentConfiguration['service']; setup: Setup; }) { - return withApmSpan('find_exact_agent_configuration', async () => { - const { internalClient, indices } = setup; - - const serviceNameFilter = service.name - ? { term: { [SERVICE_NAME]: service.name } } - : { bool: { must_not: [{ exists: { field: SERVICE_NAME } }] } }; - - const environmentFilter = service.environment - ? { term: { [SERVICE_ENVIRONMENT]: service.environment } } - : { bool: { must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }] } }; - - const params = { - index: indices.apmAgentConfigurationIndex, - body: { - query: { - bool: { filter: [serviceNameFilter, environmentFilter] }, - }, + const { internalClient, indices } = setup; + + const serviceNameFilter = service.name + ? { term: { [SERVICE_NAME]: service.name } } + : { bool: { must_not: [{ exists: { field: SERVICE_NAME } }] } }; + + const environmentFilter = service.environment + ? { term: { [SERVICE_ENVIRONMENT]: service.environment } } + : { bool: { must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }] } }; + + const params = { + index: indices.apmAgentConfigurationIndex, + body: { + query: { + bool: { filter: [serviceNameFilter, environmentFilter] }, }, - }; + }, + }; - const resp = await internalClient.search( - params - ); + const resp = await internalClient.search( + 'find_exact_agent_configuration', + params + ); - const hit = resp.hits.hits[0] as SearchHit | undefined; + const hit = resp.hits.hits[0] as SearchHit | undefined; - if (!hit) { - return; - } + if (!hit) { + return; + } - return convertConfigSettingsToString(hit); - }); + return convertConfigSettingsToString(hit); } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_agent_name_by_service.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_agent_name_by_service.ts index 379ed12e373895..0b6dd10b42e254 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_agent_name_by_service.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_agent_name_by_service.ts @@ -9,7 +9,6 @@ import { ProcessorEvent } from '../../../../common/processor_event'; import { Setup } from '../../helpers/setup_request'; import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames'; import { AGENT_NAME } from '../../../../common/elasticsearch_fieldnames'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function getAgentNameByService({ serviceName, @@ -18,35 +17,36 @@ export async function getAgentNameByService({ serviceName: string; setup: Setup; }) { - return withApmSpan('get_agent_name_by_service', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const params = { - terminateAfter: 1, - apm: { - events: [ - ProcessorEvent.transaction, - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [{ term: { [SERVICE_NAME]: serviceName } }], - }, + const params = { + terminateAfter: 1, + apm: { + events: [ + ProcessorEvent.transaction, + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [{ term: { [SERVICE_NAME]: serviceName } }], }, - aggs: { - agent_names: { - terms: { field: AGENT_NAME, size: 1 }, - }, + }, + aggs: { + agent_names: { + terms: { field: AGENT_NAME, size: 1 }, }, }, - }; + }, + }; - const { aggregations } = await apmEventClient.search(params); - const agentName = aggregations?.agent_names.buckets[0]?.key; - return agentName as string | undefined; - }); + const { aggregations } = await apmEventClient.search( + 'get_agent_name_by_service', + params + ); + const agentName = aggregations?.agent_names.buckets[0]?.key; + return agentName as string | undefined; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts index 4a32b3c3a370bd..124a373d3cf070 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_environments/get_existing_environments_for_service.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { withApmSpan } from '../../../../utils/with_apm_span'; import { Setup } from '../../../helpers/setup_request'; import { SERVICE_NAME, @@ -20,36 +19,37 @@ export async function getExistingEnvironmentsForService({ serviceName: string | undefined; setup: Setup; }) { - return withApmSpan('get_existing_environments_for_service', async () => { - const { internalClient, indices, config } = setup; - const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; + const { internalClient, indices, config } = setup; + const maxServiceEnvironments = config['xpack.apm.maxServiceEnvironments']; - const bool = serviceName - ? { filter: [{ term: { [SERVICE_NAME]: serviceName } }] } - : { must_not: [{ exists: { field: SERVICE_NAME } }] }; + const bool = serviceName + ? { filter: [{ term: { [SERVICE_NAME]: serviceName } }] } + : { must_not: [{ exists: { field: SERVICE_NAME } }] }; - const params = { - index: indices.apmAgentConfigurationIndex, - body: { - size: 0, - query: { bool }, - aggs: { - environments: { - terms: { - field: SERVICE_ENVIRONMENT, - missing: ALL_OPTION_VALUE, - size: maxServiceEnvironments, - }, + const params = { + index: indices.apmAgentConfigurationIndex, + body: { + size: 0, + query: { bool }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + missing: ALL_OPTION_VALUE, + size: maxServiceEnvironments, }, }, }, - }; + }, + }; - const resp = await internalClient.search(params); - const existingEnvironments = - resp.aggregations?.environments.buckets.map( - (bucket) => bucket.key as string - ) || []; - return existingEnvironments; - }); + const resp = await internalClient.search( + 'get_existing_environments_for_service', + params + ); + const existingEnvironments = + resp.aggregations?.environments.buckets.map( + (bucket) => bucket.key as string + ) || []; + return existingEnvironments; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_service_names.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_service_names.ts index 9c56455f45902f..0786bc6bc27714 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_service_names.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/get_service_names.ts @@ -11,52 +11,52 @@ import { PromiseReturnType } from '../../../../../observability/typings/common'; import { SERVICE_NAME } from '../../../../common/elasticsearch_fieldnames'; import { ALL_OPTION_VALUE } from '../../../../common/agent_configuration/all_option'; import { getProcessorEventForAggregatedTransactions } from '../../helpers/aggregated_transactions'; -import { withApmSpan } from '../../../utils/with_apm_span'; export type AgentConfigurationServicesAPIResponse = PromiseReturnType< typeof getServiceNames >; -export function getServiceNames({ +export async function getServiceNames({ setup, searchAggregatedTransactions, }: { setup: Setup; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_service_names_for_agent_config', async () => { - const { apmEventClient, config } = setup; - const maxServiceSelection = config['xpack.apm.maxServiceSelection']; + const { apmEventClient, config } = setup; + const maxServiceSelection = config['xpack.apm.maxServiceSelection']; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ProcessorEvent.error, - ProcessorEvent.metric, - ], - }, - body: { - timeout: '1ms', - size: 0, - aggs: { - services: { - terms: { - field: SERVICE_NAME, - size: maxServiceSelection, - min_doc_count: 0, - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + body: { + timeout: '1ms', + size: 0, + aggs: { + services: { + terms: { + field: SERVICE_NAME, + size: maxServiceSelection, + min_doc_count: 0, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); - const serviceNames = - resp.aggregations?.services.buckets - .map((bucket) => bucket.key as string) - .sort() || []; - return [ALL_OPTION_VALUE, ...serviceNames]; - }); + const resp = await apmEventClient.search( + 'get_service_names_for_agent_config', + params + ); + const serviceNames = + resp.aggregations?.services.buckets + .map((bucket) => bucket.key as string) + .sort() || []; + return [ALL_OPTION_VALUE, ...serviceNames]; } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/list_configurations.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/list_configurations.ts index adcfe88392dc8b..098888c23ccbc9 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/list_configurations.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/list_configurations.ts @@ -8,7 +8,6 @@ import { Setup } from '../../helpers/setup_request'; import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types'; import { convertConfigSettingsToString } from './convert_settings_to_string'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function listConfigurations({ setup }: { setup: Setup }) { const { internalClient, indices } = setup; @@ -18,8 +17,9 @@ export async function listConfigurations({ setup }: { setup: Setup }) { size: 200, }; - const resp = await withApmSpan('list_agent_configurations', () => - internalClient.search(params) + const resp = await internalClient.search( + 'list_agent_configuration', + params ); return resp.hits.hits diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/mark_applied_by_agent.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/mark_applied_by_agent.ts index 2026742a936a49..5fa4993921570b 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/mark_applied_by_agent.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/mark_applied_by_agent.ts @@ -29,5 +29,8 @@ export async function markAppliedByAgent({ }, }; - return internalClient.index(params); + return internalClient.index( + 'mark_configuration_applied_by_agent', + params + ); } diff --git a/x-pack/plugins/apm/server/lib/settings/agent_configuration/search_configurations.ts b/x-pack/plugins/apm/server/lib/settings/agent_configuration/search_configurations.ts index 7454128a741d5c..4e27953b3a315d 100644 --- a/x-pack/plugins/apm/server/lib/settings/agent_configuration/search_configurations.ts +++ b/x-pack/plugins/apm/server/lib/settings/agent_configuration/search_configurations.ts @@ -13,7 +13,6 @@ import { import { Setup } from '../../helpers/setup_request'; import { AgentConfiguration } from '../../../../common/agent_configuration/configuration_types'; import { convertConfigSettingsToString } from './convert_settings_to_string'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function searchConfigurations({ service, @@ -22,65 +21,64 @@ export async function searchConfigurations({ service: AgentConfiguration['service']; setup: Setup; }) { - return withApmSpan('search_agent_configurations', async () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - // In the following `constant_score` is being used to disable IDF calculation (where frequency of a term influences scoring). - // Additionally a boost has been added to service.name to ensure it scores higher. - // If there is tie between a config with a matching service.name and a config with a matching environment, the config that matches service.name wins - const serviceNameFilter = service.name - ? [ - { - constant_score: { - filter: { term: { [SERVICE_NAME]: service.name } }, - boost: 2, - }, + // In the following `constant_score` is being used to disable IDF calculation (where frequency of a term influences scoring). + // Additionally a boost has been added to service.name to ensure it scores higher. + // If there is tie between a config with a matching service.name and a config with a matching environment, the config that matches service.name wins + const serviceNameFilter = service.name + ? [ + { + constant_score: { + filter: { term: { [SERVICE_NAME]: service.name } }, + boost: 2, }, - ] - : []; + }, + ] + : []; - const environmentFilter = service.environment - ? [ - { - constant_score: { - filter: { term: { [SERVICE_ENVIRONMENT]: service.environment } }, - boost: 1, - }, + const environmentFilter = service.environment + ? [ + { + constant_score: { + filter: { term: { [SERVICE_ENVIRONMENT]: service.environment } }, + boost: 1, }, - ] - : []; + }, + ] + : []; - const params = { - index: indices.apmAgentConfigurationIndex, - body: { - query: { - bool: { - minimum_should_match: 2, - should: [ - ...serviceNameFilter, - ...environmentFilter, - { bool: { must_not: [{ exists: { field: SERVICE_NAME } }] } }, - { - bool: { - must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }], - }, + const params = { + index: indices.apmAgentConfigurationIndex, + body: { + query: { + bool: { + minimum_should_match: 2, + should: [ + ...serviceNameFilter, + ...environmentFilter, + { bool: { must_not: [{ exists: { field: SERVICE_NAME } }] } }, + { + bool: { + must_not: [{ exists: { field: SERVICE_ENVIRONMENT } }], }, - ], - }, + }, + ], }, }, - }; + }, + }; - const resp = await internalClient.search( - params - ); + const resp = await internalClient.search( + 'search_agent_configurations', + params + ); - const hit = resp.hits.hits[0] as SearchHit | undefined; + const hit = resp.hits.hits[0] as SearchHit | undefined; - if (!hit) { - return; - } + if (!hit) { + return; + } - return convertConfigSettingsToString(hit); - }); + return convertConfigSettingsToString(hit); } diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.test.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.test.ts index 47ee91232ea48e..051b9e2809e494 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.test.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.test.ts @@ -39,17 +39,20 @@ describe('Create or Update Custom link', () => { it('creates a new custom link', () => { createOrUpdateCustomLink({ customLink, setup: mockedSetup }); - expect(internalClientIndexMock).toHaveBeenCalledWith({ - refresh: true, - index: 'apmCustomLinkIndex', - body: { - '@timestamp': 1570737000000, - label: 'foo', - url: 'http://elastic.com/{{trace.id}}', - 'service.name': ['opbeans-java'], - 'transaction.type': ['Request'], - }, - }); + expect(internalClientIndexMock).toHaveBeenCalledWith( + 'create_or_update_custom_link', + { + refresh: true, + index: 'apmCustomLinkIndex', + body: { + '@timestamp': 1570737000000, + label: 'foo', + url: 'http://elastic.com/{{trace.id}}', + 'service.name': ['opbeans-java'], + 'transaction.type': ['Request'], + }, + } + ); }); it('update a new custom link', () => { createOrUpdateCustomLink({ @@ -57,17 +60,20 @@ describe('Create or Update Custom link', () => { customLink, setup: mockedSetup, }); - expect(internalClientIndexMock).toHaveBeenCalledWith({ - refresh: true, - index: 'apmCustomLinkIndex', - id: 'bar', - body: { - '@timestamp': 1570737000000, - label: 'foo', - url: 'http://elastic.com/{{trace.id}}', - 'service.name': ['opbeans-java'], - 'transaction.type': ['Request'], - }, - }); + expect(internalClientIndexMock).toHaveBeenCalledWith( + 'create_or_update_custom_link', + { + refresh: true, + index: 'apmCustomLinkIndex', + id: 'bar', + body: { + '@timestamp': 1570737000000, + label: 'foo', + url: 'http://elastic.com/{{trace.id}}', + 'service.name': ['opbeans-java'], + 'transaction.type': ['Request'], + }, + } + ); }); }); diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.ts index 7e546fb5550360..8f14e87fe183bc 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/create_or_update_custom_link.ts @@ -12,7 +12,6 @@ import { import { Setup } from '../../helpers/setup_request'; import { toESFormat } from './helper'; import { APMIndexDocumentParams } from '../../helpers/create_es_client/create_internal_es_client'; -import { withApmSpan } from '../../../utils/with_apm_span'; export function createOrUpdateCustomLink({ customLinkId, @@ -23,23 +22,21 @@ export function createOrUpdateCustomLink({ customLink: Omit; setup: Setup; }) { - return withApmSpan('create_or_update_custom_link', () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - const params: APMIndexDocumentParams = { - refresh: true, - index: indices.apmCustomLinkIndex, - body: { - '@timestamp': Date.now(), - ...toESFormat(customLink), - }, - }; + const params: APMIndexDocumentParams = { + refresh: true, + index: indices.apmCustomLinkIndex, + body: { + '@timestamp': Date.now(), + ...toESFormat(customLink), + }, + }; - // by specifying an id elasticsearch will delete the previous doc and insert the updated doc - if (customLinkId) { - params.id = customLinkId; - } + // by specifying an id elasticsearch will delete the previous doc and insert the updated doc + if (customLinkId) { + params.id = customLinkId; + } - return internalClient.index(params); - }); + return internalClient.index('create_or_update_custom_link', params); } diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts index 7c88bcc43cc7f7..bf7cfb33d87ac6 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/delete_custom_link.ts @@ -5,7 +5,6 @@ * 2.0. */ -import { withApmSpan } from '../../../utils/with_apm_span'; import { Setup } from '../../helpers/setup_request'; export function deleteCustomLink({ @@ -15,15 +14,13 @@ export function deleteCustomLink({ customLinkId: string; setup: Setup; }) { - return withApmSpan('delete_custom_link', () => { - const { internalClient, indices } = setup; + const { internalClient, indices } = setup; - const params = { - refresh: 'wait_for' as const, - index: indices.apmCustomLinkIndex, - id: customLinkId, - }; + const params = { + refresh: 'wait_for' as const, + index: indices.apmCustomLinkIndex, + id: customLinkId, + }; - return internalClient.delete(params); - }); + return internalClient.delete('delete_custom_link', params); } diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/get_transaction.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/get_transaction.ts index d3d9b452853540..91bc8c85bc0143 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/get_transaction.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/get_transaction.ts @@ -11,43 +11,43 @@ import { Setup } from '../../helpers/setup_request'; import { ProcessorEvent } from '../../../../common/processor_event'; import { filterOptionsRt } from './custom_link_types'; import { splitFilterValueByComma } from './helper'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getTransaction({ +export async function getTransaction({ setup, filters = {}, }: { setup: Setup; filters?: t.TypeOf; }) { - return withApmSpan('get_transaction_for_custom_link', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const esFilters = compact( - Object.entries(filters) - // loops through the filters splitting the value by comma and removing white spaces - .map(([key, value]) => { - if (value) { - return { terms: { [key]: splitFilterValueByComma(value) } }; - } - }) - ); + const esFilters = compact( + Object.entries(filters) + // loops through the filters splitting the value by comma and removing white spaces + .map(([key, value]) => { + if (value) { + return { terms: { [key]: splitFilterValueByComma(value) } }; + } + }) + ); - const params = { - terminateAfter: 1, - apm: { - events: [ProcessorEvent.transaction as const], - }, - size: 1, - body: { - query: { - bool: { - filter: esFilters, - }, + const params = { + terminateAfter: 1, + apm: { + events: [ProcessorEvent.transaction as const], + }, + size: 1, + body: { + query: { + bool: { + filter: esFilters, }, }, - }; - const resp = await apmEventClient.search(params); - return resp.hits.hits[0]?._source; - }); + }, + }; + const resp = await apmEventClient.search( + 'get_transaction_for_custom_link', + params + ); + return resp.hits.hits[0]?._source; } diff --git a/x-pack/plugins/apm/server/lib/settings/custom_link/list_custom_links.ts b/x-pack/plugins/apm/server/lib/settings/custom_link/list_custom_links.ts index 0eac2e08d0901e..d477da85e0d9b8 100644 --- a/x-pack/plugins/apm/server/lib/settings/custom_link/list_custom_links.ts +++ b/x-pack/plugins/apm/server/lib/settings/custom_link/list_custom_links.ts @@ -14,54 +14,54 @@ import { import { Setup } from '../../helpers/setup_request'; import { fromESFormat } from './helper'; import { filterOptionsRt } from './custom_link_types'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function listCustomLinks({ +export async function listCustomLinks({ setup, filters = {}, }: { setup: Setup; filters?: t.TypeOf; }): Promise { - return withApmSpan('list_custom_links', async () => { - const { internalClient, indices } = setup; - const esFilters = Object.entries(filters).map(([key, value]) => { - return { + const { internalClient, indices } = setup; + const esFilters = Object.entries(filters).map(([key, value]) => { + return { + bool: { + minimum_should_match: 1, + should: [ + { term: { [key]: value } }, + { bool: { must_not: [{ exists: { field: key } }] } }, + ] as QueryDslQueryContainer[], + }, + }; + }); + + const params = { + index: indices.apmCustomLinkIndex, + size: 500, + body: { + query: { bool: { - minimum_should_match: 1, - should: [ - { term: { [key]: value } }, - { bool: { must_not: [{ exists: { field: key } }] } }, - ] as QueryDslQueryContainer[], + filter: esFilters, }, - }; - }); - - const params = { - index: indices.apmCustomLinkIndex, - size: 500, - body: { - query: { - bool: { - filter: esFilters, + }, + sort: [ + { + 'label.keyword': { + order: 'asc' as const, }, }, - sort: [ - { - 'label.keyword': { - order: 'asc' as const, - }, - }, - ], - }, - }; - const resp = await internalClient.search(params); - const customLinks = resp.hits.hits.map((item) => - fromESFormat({ - id: item._id, - ...item._source, - }) - ); - return customLinks; - }); + ], + }, + }; + const resp = await internalClient.search( + 'list_custom_links', + params + ); + const customLinks = resp.hits.hits.map((item) => + fromESFormat({ + id: item._id, + ...item._source, + }) + ); + return customLinks; } diff --git a/x-pack/plugins/apm/server/lib/traces/get_trace_items.ts b/x-pack/plugins/apm/server/lib/traces/get_trace_items.ts index 157f09978eaec6..68d316ef55df98 100644 --- a/x-pack/plugins/apm/server/lib/traces/get_trace_items.ts +++ b/x-pack/plugins/apm/server/lib/traces/get_trace_items.ts @@ -19,7 +19,6 @@ import { APMError } from '../../../typings/es_schemas/ui/apm_error'; import { rangeQuery } from '../../../server/utils/queries'; import { Setup, SetupTimeRange } from '../helpers/setup_request'; import { PromiseValueType } from '../../../typings/common'; -import { withApmSpan } from '../../utils/with_apm_span'; export interface ErrorsPerTransaction { [transactionId: string]: number; @@ -29,103 +28,94 @@ export async function getTraceItems( traceId: string, setup: Setup & SetupTimeRange ) { - return withApmSpan('get_trace_items', async () => { - const { start, end, apmEventClient, config } = setup; - const maxTraceItems = config['xpack.apm.ui.maxTraceItems']; - const excludedLogLevels = ['debug', 'info', 'warning']; + const { start, end, apmEventClient, config } = setup; + const maxTraceItems = config['xpack.apm.ui.maxTraceItems']; + const excludedLogLevels = ['debug', 'info', 'warning']; - const errorResponsePromise = withApmSpan('get_trace_error_items', () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.error], + const errorResponsePromise = apmEventClient.search('get_trace_items', { + apm: { + events: [ProcessorEvent.error], + }, + body: { + size: maxTraceItems, + query: { + bool: { + filter: [ + { term: { [TRACE_ID]: traceId } }, + ...rangeQuery(start, end), + ], + must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } }, }, - body: { - size: maxTraceItems, - query: { - bool: { - filter: [ - { term: { [TRACE_ID]: traceId } }, - ...rangeQuery(start, end), - ], - must_not: { terms: { [ERROR_LOG_LEVEL]: excludedLogLevels } }, - }, - }, - aggs: { - by_transaction_id: { - terms: { - field: TRANSACTION_ID, - size: maxTraceItems, - // high cardinality - execution_hint: 'map' as const, - }, - }, + }, + aggs: { + by_transaction_id: { + terms: { + field: TRANSACTION_ID, + size: maxTraceItems, + // high cardinality + execution_hint: 'map' as const, }, }, - }) - ); + }, + }, + }); - const traceResponsePromise = withApmSpan('get_trace_span_items', () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.span, ProcessorEvent.transaction], - }, - body: { - size: maxTraceItems, - query: { - bool: { - filter: [ - { term: { [TRACE_ID]: traceId } }, - ...rangeQuery(start, end), - ] as QueryDslQueryContainer[], - should: { - exists: { field: PARENT_ID }, - }, - }, + const traceResponsePromise = apmEventClient.search('get_trace_span_items', { + apm: { + events: [ProcessorEvent.span, ProcessorEvent.transaction], + }, + body: { + size: maxTraceItems, + query: { + bool: { + filter: [ + { term: { [TRACE_ID]: traceId } }, + ...rangeQuery(start, end), + ] as QueryDslQueryContainer[], + should: { + exists: { field: PARENT_ID }, }, - sort: [ - { _score: { order: 'asc' as const } }, - { [TRANSACTION_DURATION]: { order: 'desc' as const } }, - { [SPAN_DURATION]: { order: 'desc' as const } }, - ], - track_total_hits: true, }, - }) - ); + }, + sort: [ + { _score: { order: 'asc' as const } }, + { [TRANSACTION_DURATION]: { order: 'desc' as const } }, + { [SPAN_DURATION]: { order: 'desc' as const } }, + ], + track_total_hits: true, + }, + }); - const [errorResponse, traceResponse]: [ - // explicit intermediary types to avoid TS "excessively deep" error - PromiseValueType, - PromiseValueType - ] = (await Promise.all([ - errorResponsePromise, - traceResponsePromise, - ])) as any; + const [errorResponse, traceResponse]: [ + // explicit intermediary types to avoid TS "excessively deep" error + PromiseValueType, + PromiseValueType + ] = (await Promise.all([errorResponsePromise, traceResponsePromise])) as any; - const exceedsMax = traceResponse.hits.total.value > maxTraceItems; + const exceedsMax = traceResponse.hits.total.value > maxTraceItems; - const items = traceResponse.hits.hits.map((hit) => hit._source); + const items = traceResponse.hits.hits.map((hit) => hit._source); - const errorFrequencies: { - errorsPerTransaction: ErrorsPerTransaction; - errorDocs: APMError[]; - } = { - errorDocs: errorResponse.hits.hits.map(({ _source }) => _source), - errorsPerTransaction: - errorResponse.aggregations?.by_transaction_id.buckets.reduce( - (acc, current) => { - return { - ...acc, - [current.key]: current.doc_count, - }; - }, - {} as ErrorsPerTransaction - ) ?? {}, - }; + const errorFrequencies: { + errorsPerTransaction: ErrorsPerTransaction; + errorDocs: APMError[]; + } = { + errorDocs: errorResponse.hits.hits.map(({ _source }) => _source), + errorsPerTransaction: + errorResponse.aggregations?.by_transaction_id.buckets.reduce( + (acc, current) => { + return { + ...acc, + [current.key]: current.doc_count, + }; + }, + {} as ErrorsPerTransaction + ) ?? {}, + }; - return { - items, - exceedsMax, - ...errorFrequencies, - }; - }); + return { + items, + exceedsMax, + ...errorFrequencies, + }; } diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts b/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts index 71f803a03bf85f..6499e80be93027 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/get_error_rate.ts @@ -31,7 +31,6 @@ import { getOutcomeAggregation, getTransactionErrorRateTimeSeries, } from '../helpers/transaction_error_rate'; -import { withApmSpan } from '../../utils/with_apm_span'; export async function getErrorRate({ environment, @@ -58,81 +57,82 @@ export async function getErrorRate({ transactionErrorRate: Coordinate[]; average: number | null; }> { - return withApmSpan('get_transaction_group_error_rate', async () => { - const { apmEventClient } = setup; - - const transactionNamefilter = transactionName - ? [{ term: { [TRANSACTION_NAME]: transactionName } }] - : []; - const transactionTypefilter = transactionType - ? [{ term: { [TRANSACTION_TYPE]: transactionType } }] - : []; - - const filter = [ - { term: { [SERVICE_NAME]: serviceName } }, - { - terms: { - [EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success], - }, - }, - ...transactionNamefilter, - ...transactionTypefilter, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ]; - - const outcomes = getOutcomeAggregation(); - - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], + const { apmEventClient } = setup; + + const transactionNamefilter = transactionName + ? [{ term: { [TRANSACTION_NAME]: transactionName } }] + : []; + const transactionTypefilter = transactionType + ? [{ term: { [TRANSACTION_TYPE]: transactionType } }] + : []; + + const filter = [ + { term: { [SERVICE_NAME]: serviceName } }, + { + terms: { + [EVENT_OUTCOME]: [EventOutcome.failure, EventOutcome.success], }, - body: { - size: 0, - query: { bool: { filter } }, - aggs: { - outcomes, - timeseries: { - date_histogram: { - field: '@timestamp', - fixed_interval: getBucketSize({ start, end }).intervalString, - min_doc_count: 0, - extended_bounds: { min: start, max: end }, - }, - aggs: { - outcomes, - }, + }, + ...transactionNamefilter, + ...transactionTypefilter, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ]; + + const outcomes = getOutcomeAggregation(); + + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { bool: { filter } }, + aggs: { + outcomes, + timeseries: { + date_histogram: { + field: '@timestamp', + fixed_interval: getBucketSize({ start, end }).intervalString, + min_doc_count: 0, + extended_bounds: { min: start, max: end }, + }, + aggs: { + outcomes, }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); + const resp = await apmEventClient.search( + 'get_transaction_group_error_rate', + params + ); - const noHits = resp.hits.total.value === 0; + const noHits = resp.hits.total.value === 0; - if (!resp.aggregations) { - return { noHits, transactionErrorRate: [], average: null }; - } + if (!resp.aggregations) { + return { noHits, transactionErrorRate: [], average: null }; + } - const transactionErrorRate = getTransactionErrorRateTimeSeries( - resp.aggregations.timeseries.buckets - ); + const transactionErrorRate = getTransactionErrorRateTimeSeries( + resp.aggregations.timeseries.buckets + ); - const average = calculateTransactionErrorPercentage( - resp.aggregations.outcomes - ); + const average = calculateTransactionErrorPercentage( + resp.aggregations.outcomes + ); - return { noHits, transactionErrorRate, average }; - }); + return { noHits, transactionErrorRate, average }; } export async function getErrorRatePeriods({ diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts b/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts index 8156d52d984dfe..34fd86f2fc5989 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/get_transaction_group_stats.ts @@ -11,7 +11,6 @@ import { TRANSACTION_TYPE } from '../../../common/elasticsearch_fieldnames'; import { arrayUnionToCallable } from '../../../common/utils/array_union_to_callable'; import { TransactionGroupRequestBase, TransactionGroupSetup } from './fetcher'; import { getTransactionDurationFieldForAggregatedTransactions } from '../helpers/aggregated_transactions'; -import { withApmSpan } from '../../utils/with_apm_span'; interface MetricParams { request: TransactionGroupRequestBase; @@ -39,124 +38,128 @@ function mergeRequestWithAggs< }); } -export function getAverages({ +export async function getAverages({ request, setup, searchAggregatedTransactions, }: MetricParams) { - return withApmSpan('get_avg_transaction_group_duration', async () => { - const params = mergeRequestWithAggs(request, { + const params = mergeRequestWithAggs(request, { + avg: { avg: { - avg: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, - }); - - const response = await setup.apmEventClient.search(params); - - return arrayUnionToCallable( - response.aggregations?.transaction_groups.buckets ?? [] - ).map((bucket) => { - return { - key: bucket.key as BucketKey, - avg: bucket.avg.value, - }; - }); + }, + }); + + const response = await setup.apmEventClient.search( + 'get_avg_transaction_group_duration', + params + ); + + return arrayUnionToCallable( + response.aggregations?.transaction_groups.buckets ?? [] + ).map((bucket) => { + return { + key: bucket.key as BucketKey, + avg: bucket.avg.value, + }; }); } -export function getCounts({ request, setup }: MetricParams) { - return withApmSpan('get_transaction_group_transaction_count', async () => { - const params = mergeRequestWithAggs(request, { - transaction_type: { - top_metrics: { - sort: { - '@timestamp': 'desc' as const, - }, - metrics: [ - { - field: TRANSACTION_TYPE, - } as const, - ], +export async function getCounts({ request, setup }: MetricParams) { + const params = mergeRequestWithAggs(request, { + transaction_type: { + top_metrics: { + sort: { + '@timestamp': 'desc' as const, }, + metrics: [ + { + field: TRANSACTION_TYPE, + } as const, + ], }, - }); - - const response = await setup.apmEventClient.search(params); - - return arrayUnionToCallable( - response.aggregations?.transaction_groups.buckets ?? [] - ).map((bucket) => { - return { - key: bucket.key as BucketKey, - count: bucket.doc_count, - transactionType: bucket.transaction_type.top[0].metrics[ - TRANSACTION_TYPE - ] as string, - }; - }); + }, + }); + + const response = await setup.apmEventClient.search( + 'get_transaction_group_transaction_count', + params + ); + + return arrayUnionToCallable( + response.aggregations?.transaction_groups.buckets ?? [] + ).map((bucket) => { + return { + key: bucket.key as BucketKey, + count: bucket.doc_count, + transactionType: bucket.transaction_type.top[0].metrics[ + TRANSACTION_TYPE + ] as string, + }; }); } -export function getSums({ +export async function getSums({ request, setup, searchAggregatedTransactions, }: MetricParams) { - return withApmSpan('get_transaction_group_latency_sums', async () => { - const params = mergeRequestWithAggs(request, { + const params = mergeRequestWithAggs(request, { + sum: { sum: { - sum: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, - }); - - const response = await setup.apmEventClient.search(params); - - return arrayUnionToCallable( - response.aggregations?.transaction_groups.buckets ?? [] - ).map((bucket) => { - return { - key: bucket.key as BucketKey, - sum: bucket.sum.value, - }; - }); + }, + }); + + const response = await setup.apmEventClient.search( + 'get_transaction_group_latency_sums', + params + ); + + return arrayUnionToCallable( + response.aggregations?.transaction_groups.buckets ?? [] + ).map((bucket) => { + return { + key: bucket.key as BucketKey, + sum: bucket.sum.value, + }; }); } -export function getPercentiles({ +export async function getPercentiles({ request, setup, searchAggregatedTransactions, }: MetricParams) { - return withApmSpan('get_transaction_group_latency_percentiles', async () => { - const params = mergeRequestWithAggs(request, { - p95: { - percentiles: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - hdr: { number_of_significant_value_digits: 2 }, - percents: [95], - }, + const params = mergeRequestWithAggs(request, { + p95: { + percentiles: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), + hdr: { number_of_significant_value_digits: 2 }, + percents: [95], }, - }); - - const response = await setup.apmEventClient.search(params); - - return arrayUnionToCallable( - response.aggregations?.transaction_groups.buckets ?? [] - ).map((bucket) => { - return { - key: bucket.key as BucketKey, - p95: Object.values(bucket.p95.values)[0], - }; - }); + }, + }); + + const response = await setup.apmEventClient.search( + 'get_transaction_group_latency_percentiles', + params + ); + + return arrayUnionToCallable( + response.aggregations?.transaction_groups.buckets ?? [] + ).map((bucket) => { + return { + key: bucket.key as BucketKey, + p95: Object.values(bucket.p95.values)[0], + }; }); } diff --git a/x-pack/plugins/apm/server/lib/transaction_groups/queries.test.ts b/x-pack/plugins/apm/server/lib/transaction_groups/queries.test.ts index 6cb85b62205b87..5c1754cd36ef4c 100644 --- a/x-pack/plugins/apm/server/lib/transaction_groups/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/transaction_groups/queries.test.ts @@ -33,7 +33,7 @@ describe('transaction group queries', () => { ) ); - const allParams = mock.spy.mock.calls.map((call) => call[0]); + const allParams = mock.spy.mock.calls.map((call) => call[1]); expect(allParams).toMatchSnapshot(); }); @@ -51,7 +51,7 @@ describe('transaction group queries', () => { ) ); - const allParams = mock.spy.mock.calls.map((call) => call[0]); + const allParams = mock.spy.mock.calls.map((call) => call[1]); expect(allParams).toMatchSnapshot(); }); diff --git a/x-pack/plugins/apm/server/lib/transactions/breakdown/index.ts b/x-pack/plugins/apm/server/lib/transactions/breakdown/index.ts index 568769b52e2b4b..20534a5fa7cbfc 100644 --- a/x-pack/plugins/apm/server/lib/transactions/breakdown/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/breakdown/index.ts @@ -26,9 +26,8 @@ import { import { getMetricsDateHistogramParams } from '../../helpers/metrics'; import { MAX_KPIS } from './constants'; import { getVizColorForIndex } from '../../../../common/viz_colors'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getTransactionBreakdown({ +export async function getTransactionBreakdown({ environment, kuery, setup, @@ -43,205 +42,203 @@ export function getTransactionBreakdown({ transactionName?: string; transactionType: string; }) { - return withApmSpan('get_transaction_breakdown', async () => { - const { apmEventClient, start, end, config } = setup; + const { apmEventClient, start, end, config } = setup; - const subAggs = { - sum_all_self_times: { - sum: { - field: SPAN_SELF_TIME_SUM, - }, + const subAggs = { + sum_all_self_times: { + sum: { + field: SPAN_SELF_TIME_SUM, }, - total_transaction_breakdown_count: { - sum: { - field: TRANSACTION_BREAKDOWN_COUNT, - }, + }, + total_transaction_breakdown_count: { + sum: { + field: TRANSACTION_BREAKDOWN_COUNT, }, - types: { - terms: { - field: SPAN_TYPE, - size: 20, - order: { - _count: 'desc' as const, - }, + }, + types: { + terms: { + field: SPAN_TYPE, + size: 20, + order: { + _count: 'desc' as const, }, - aggs: { - subtypes: { - terms: { - field: SPAN_SUBTYPE, - missing: '', - size: 20, - order: { - _count: 'desc' as const, - }, + }, + aggs: { + subtypes: { + terms: { + field: SPAN_SUBTYPE, + missing: '', + size: 20, + order: { + _count: 'desc' as const, }, - aggs: { - total_self_time_per_subtype: { - sum: { - field: SPAN_SELF_TIME_SUM, - }, + }, + aggs: { + total_self_time_per_subtype: { + sum: { + field: SPAN_SELF_TIME_SUM, }, }, }, }, }, - }; - - const filters = [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - { + }, + }; + + const filters = [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + { + bool: { + should: [ + { exists: { field: SPAN_SELF_TIME_SUM } }, + { exists: { field: TRANSACTION_BREAKDOWN_COUNT } }, + ], + minimum_should_match: 1, + }, + }, + ]; + + if (transactionName) { + filters.push({ term: { [TRANSACTION_NAME]: transactionName } }); + } + + const params = { + apm: { + events: [ProcessorEvent.metric], + }, + body: { + size: 0, + query: { bool: { - should: [ - { exists: { field: SPAN_SELF_TIME_SUM } }, - { exists: { field: TRANSACTION_BREAKDOWN_COUNT } }, - ], - minimum_should_match: 1, + filter: filters, }, }, - ]; - - if (transactionName) { - filters.push({ term: { [TRANSACTION_NAME]: transactionName } }); - } - - const params = { - apm: { - events: [ProcessorEvent.metric], - }, - body: { - size: 0, - query: { - bool: { - filter: filters, - }, - }, - aggs: { - ...subAggs, - by_date: { - date_histogram: getMetricsDateHistogramParams( - start, - end, - config['xpack.apm.metricsInterval'] - ), - aggs: subAggs, - }, + aggs: { + ...subAggs, + by_date: { + date_histogram: getMetricsDateHistogramParams( + start, + end, + config['xpack.apm.metricsInterval'] + ), + aggs: subAggs, }, }, + }, + }; + + const resp = await apmEventClient.search('get_transaction_breakdown', params); + + const formatBucket = ( + aggs: + | Required['aggregations'] + | Required['aggregations']['by_date']['buckets'][0] + ) => { + const sumAllSelfTimes = aggs.sum_all_self_times.value || 0; + + const breakdowns = flatten( + aggs.types.buckets.map((bucket) => { + const type = bucket.key as string; + + return bucket.subtypes.buckets.map((subBucket) => { + return { + name: (subBucket.key as string) || type, + percentage: + (subBucket.total_self_time_per_subtype.value || 0) / + sumAllSelfTimes, + }; + }); + }) + ); + + return breakdowns; + }; + + const visibleKpis = resp.aggregations + ? orderBy(formatBucket(resp.aggregations), 'percentage', 'desc').slice( + 0, + MAX_KPIS + ) + : []; + + const kpis = orderBy( + visibleKpis.map((kpi) => ({ + ...kpi, + lowerCaseName: kpi.name.toLowerCase(), + })), + 'lowerCaseName' + ).map((kpi, index) => { + const { lowerCaseName, ...rest } = kpi; + return { + ...rest, + color: getVizColorForIndex(index), }; + }); - const resp = await apmEventClient.search(params); - - const formatBucket = ( - aggs: - | Required['aggregations'] - | Required['aggregations']['by_date']['buckets'][0] - ) => { - const sumAllSelfTimes = aggs.sum_all_self_times.value || 0; - - const breakdowns = flatten( - aggs.types.buckets.map((bucket) => { - const type = bucket.key as string; - - return bucket.subtypes.buckets.map((subBucket) => { - return { - name: (subBucket.key as string) || type, - percentage: - (subBucket.total_self_time_per_subtype.value || 0) / - sumAllSelfTimes, - }; - }); - }) - ); - - return breakdowns; - }; - - const visibleKpis = resp.aggregations - ? orderBy(formatBucket(resp.aggregations), 'percentage', 'desc').slice( - 0, - MAX_KPIS - ) - : []; - - const kpis = orderBy( - visibleKpis.map((kpi) => ({ - ...kpi, - lowerCaseName: kpi.name.toLowerCase(), - })), - 'lowerCaseName' - ).map((kpi, index) => { - const { lowerCaseName, ...rest } = kpi; - return { - ...rest, - color: getVizColorForIndex(index), - }; - }); - - const kpiNames = kpis.map((kpi) => kpi.name); + const kpiNames = kpis.map((kpi) => kpi.name); - const bucketsByDate = resp.aggregations?.by_date.buckets || []; + const bucketsByDate = resp.aggregations?.by_date.buckets || []; - const timeseriesPerSubtype = bucketsByDate.reduce((prev, bucket) => { - const formattedValues = formatBucket(bucket); - const time = bucket.key; + const timeseriesPerSubtype = bucketsByDate.reduce((prev, bucket) => { + const formattedValues = formatBucket(bucket); + const time = bucket.key; - const updatedSeries = kpiNames.reduce((p, kpiName) => { - const { name, percentage } = formattedValues.find( - (val) => val.name === kpiName - ) || { - name: kpiName, - percentage: null, - }; + const updatedSeries = kpiNames.reduce((p, kpiName) => { + const { name, percentage } = formattedValues.find( + (val) => val.name === kpiName + ) || { + name: kpiName, + percentage: null, + }; - if (!p[name]) { - p[name] = []; - } - return { - ...p, - [name]: p[name].concat({ - x: time, - y: percentage, - }), - }; - }, prev); - - const lastValues = Object.values(updatedSeries).map(last); - - // If for a given timestamp, some series have data, but others do not, - // we have to set any null values to 0 to make sure the stacked area chart - // is drawn correctly. - // If we set all values to 0, the chart always displays null values as 0, - // and the chart looks weird. - const hasAnyValues = lastValues.some((value) => value?.y !== null); - const hasNullValues = lastValues.some((value) => value?.y === null); - - if (hasAnyValues && hasNullValues) { - Object.values(updatedSeries).forEach((series) => { - const value = series[series.length - 1]; - const isEmpty = value.y === null; - if (isEmpty) { - // local mutation to prevent complicated map/reduce calls - value.y = 0; - } - }); + if (!p[name]) { + p[name] = []; } + return { + ...p, + [name]: p[name].concat({ + x: time, + y: percentage, + }), + }; + }, prev); + + const lastValues = Object.values(updatedSeries).map(last); + + // If for a given timestamp, some series have data, but others do not, + // we have to set any null values to 0 to make sure the stacked area chart + // is drawn correctly. + // If we set all values to 0, the chart always displays null values as 0, + // and the chart looks weird. + const hasAnyValues = lastValues.some((value) => value?.y !== null); + const hasNullValues = lastValues.some((value) => value?.y === null); + + if (hasAnyValues && hasNullValues) { + Object.values(updatedSeries).forEach((series) => { + const value = series[series.length - 1]; + const isEmpty = value.y === null; + if (isEmpty) { + // local mutation to prevent complicated map/reduce calls + value.y = 0; + } + }); + } - return updatedSeries; - }, {} as Record>); + return updatedSeries; + }, {} as Record>); - const timeseries = kpis.map((kpi) => ({ - title: kpi.name, - color: kpi.color, - type: 'areaStacked', - data: timeseriesPerSubtype[kpi.name], - hideLegend: false, - legendValue: asPercent(kpi.percentage, 1), - })); + const timeseries = kpis.map((kpi) => ({ + title: kpi.name, + color: kpi.color, + type: 'areaStacked', + data: timeseriesPerSubtype[kpi.name], + hideLegend: false, + legendValue: asPercent(kpi.percentage, 1), + })); - return { timeseries }; - }); + return { timeseries }; } diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts index 3b4319c37996dc..6259bb75386fb3 100644 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/distribution/get_buckets/index.ts @@ -89,48 +89,47 @@ export async function getBuckets({ ] as QueryDslQueryContainer[]; async function getSamplesForDistributionBuckets() { - const response = await withApmSpan( + const response = await apmEventClient.search( 'get_samples_for_latency_distribution_buckets', - () => - apmEventClient.search({ - apm: { - events: [ProcessorEvent.transaction], - }, - body: { - query: { - bool: { - filter: [ - ...commonFilters, - { term: { [TRANSACTION_SAMPLED]: true } }, - ], - should: [ - { term: { [TRACE_ID]: traceId } }, - { term: { [TRANSACTION_ID]: transactionId } }, - ] as QueryDslQueryContainer[], - }, + { + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + query: { + bool: { + filter: [ + ...commonFilters, + { term: { [TRANSACTION_SAMPLED]: true } }, + ], + should: [ + { term: { [TRACE_ID]: traceId } }, + { term: { [TRANSACTION_ID]: transactionId } }, + ] as QueryDslQueryContainer[], }, - aggs: { - distribution: { - histogram: getHistogramAggOptions({ - bucketSize, - field: TRANSACTION_DURATION, - distributionMax, - }), - aggs: { - samples: { - top_hits: { - _source: [TRANSACTION_ID, TRACE_ID], - size: 10, - sort: { - _score: 'desc' as const, - }, + }, + aggs: { + distribution: { + histogram: getHistogramAggOptions({ + bucketSize, + field: TRANSACTION_DURATION, + distributionMax, + }), + aggs: { + samples: { + top_hits: { + _source: [TRANSACTION_ID, TRACE_ID], + size: 10, + sort: { + _score: 'desc' as const, }, }, }, }, }, }, - }) + }, + } ); return ( @@ -148,41 +147,40 @@ export async function getBuckets({ } async function getDistributionBuckets() { - const response = await withApmSpan( + const response = await apmEventClient.search( 'get_latency_distribution_buckets', - () => - apmEventClient.search({ - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - query: { - bool: { - filter: [ - ...commonFilters, - ...getDocumentTypeFilterForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, + { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + query: { + bool: { + filter: [ + ...commonFilters, + ...getDocumentTypeFilterForAggregatedTransactions( + searchAggregatedTransactions + ), + ], }, - aggs: { - distribution: { - histogram: getHistogramAggOptions({ - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - bucketSize, - distributionMax, - }), - }, + }, + aggs: { + distribution: { + histogram: getHistogramAggOptions({ + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), + bucketSize, + distributionMax, + }), }, }, - }) + }, + } ); return ( diff --git a/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts b/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts index 2e86f6bb84c815..f3d4e8f6dd92d5 100644 --- a/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts +++ b/x-pack/plugins/apm/server/lib/transactions/distribution/get_distribution_max.ts @@ -20,7 +20,6 @@ import { rangeQuery, kqlQuery, } from '../../../../server/utils/queries'; -import { withApmSpan } from '../../../utils/with_apm_span'; export async function getDistributionMax({ environment, @@ -39,44 +38,45 @@ export async function getDistributionMax({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_latency_distribution_max', async () => { - const { start, end, apmEventClient } = setup; + const { start, end, apmEventClient } = setup; - const params = { - apm: { - events: [ - getProcessorEventForAggregatedTransactions( - searchAggregatedTransactions - ), - ], - }, - body: { - size: 0, - query: { - bool: { - filter: [ - { term: { [SERVICE_NAME]: serviceName } }, - { term: { [TRANSACTION_TYPE]: transactionType } }, - { term: { [TRANSACTION_NAME]: transactionName } }, - ...rangeQuery(start, end), - ...environmentQuery(environment), - ...kqlQuery(kuery), - ], - }, + const params = { + apm: { + events: [ + getProcessorEventForAggregatedTransactions( + searchAggregatedTransactions + ), + ], + }, + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [SERVICE_NAME]: serviceName } }, + { term: { [TRANSACTION_TYPE]: transactionType } }, + { term: { [TRANSACTION_NAME]: transactionName } }, + ...rangeQuery(start, end), + ...environmentQuery(environment), + ...kqlQuery(kuery), + ], }, - aggs: { - stats: { - max: { - field: getTransactionDurationFieldForAggregatedTransactions( - searchAggregatedTransactions - ), - }, + }, + aggs: { + stats: { + max: { + field: getTransactionDurationFieldForAggregatedTransactions( + searchAggregatedTransactions + ), }, }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); - return resp.aggregations?.stats.value ?? null; - }); + const resp = await apmEventClient.search( + 'get_latency_distribution_max', + params + ); + return resp.aggregations?.stats.value ?? null; } diff --git a/x-pack/plugins/apm/server/lib/transactions/get_latency_charts/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_latency_charts/index.ts index 5a583605978282..2d350090fa28b7 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_latency_charts/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_latency_charts/index.ts @@ -26,7 +26,6 @@ import { } from '../../../lib/helpers/aggregated_transactions'; import { getBucketSize } from '../../../lib/helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../../lib/helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getLatencyAggregation, getLatencyValue, @@ -112,10 +111,10 @@ function searchLatency({ }, }; - return apmEventClient.search(params); + return apmEventClient.search('get_latency_charts', params); } -export function getLatencyTimeseries({ +export async function getLatencyTimeseries({ environment, kuery, serviceName, @@ -138,40 +137,38 @@ export function getLatencyTimeseries({ start: number; end: number; }) { - return withApmSpan('get_latency_charts', async () => { - const response = await searchLatency({ - environment, - kuery, - serviceName, - transactionType, - transactionName, - setup, - searchAggregatedTransactions, - latencyAggregationType, - start, - end, - }); - - if (!response.aggregations) { - return { latencyTimeseries: [], overallAvgDuration: null }; - } - - return { - overallAvgDuration: - response.aggregations.overall_avg_duration.value || null, - latencyTimeseries: response.aggregations.latencyTimeseries.buckets.map( - (bucket) => { - return { - x: bucket.key, - y: getLatencyValue({ - latencyAggregationType, - aggregation: bucket.latency, - }), - }; - } - ), - }; + const response = await searchLatency({ + environment, + kuery, + serviceName, + transactionType, + transactionName, + setup, + searchAggregatedTransactions, + latencyAggregationType, + start, + end, }); + + if (!response.aggregations) { + return { latencyTimeseries: [], overallAvgDuration: null }; + } + + return { + overallAvgDuration: + response.aggregations.overall_avg_duration.value || null, + latencyTimeseries: response.aggregations.latencyTimeseries.buckets.map( + (bucket) => { + return { + x: bucket.key, + y: getLatencyValue({ + latencyAggregationType, + aggregation: bucket.latency, + }), + }; + } + ), + }; } export async function getLatencyPeriods({ diff --git a/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts index a0225eb47e584b..f4d9236395252c 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_throughput_charts/index.ts @@ -24,7 +24,6 @@ import { } from '../../../lib/helpers/aggregated_transactions'; import { getBucketSize } from '../../../lib/helpers/get_bucket_size'; import { Setup, SetupTimeRange } from '../../../lib/helpers/setup_request'; -import { withApmSpan } from '../../../utils/with_apm_span'; import { getThroughputBuckets } from './transform'; export type ThroughputChartsResponse = PromiseReturnType< @@ -96,7 +95,7 @@ function searchThroughput({ }, }; - return apmEventClient.search(params); + return apmEventClient.search('get_transaction_throughput_series', params); } export async function getThroughputCharts({ @@ -116,26 +115,24 @@ export async function getThroughputCharts({ setup: Setup & SetupTimeRange; searchAggregatedTransactions: boolean; }) { - return withApmSpan('get_transaction_throughput_series', async () => { - const { bucketSize, intervalString } = getBucketSize(setup); + const { bucketSize, intervalString } = getBucketSize(setup); - const response = await searchThroughput({ - environment, - kuery, - serviceName, - transactionType, - transactionName, - setup, - searchAggregatedTransactions, - intervalString, - }); - - return { - throughputTimeseries: getThroughputBuckets({ - throughputResultBuckets: response.aggregations?.throughput.buckets, - bucketSize, - setupTimeRange: setup, - }), - }; + const response = await searchThroughput({ + environment, + kuery, + serviceName, + transactionType, + transactionName, + setup, + searchAggregatedTransactions, + intervalString, }); + + return { + throughputTimeseries: getThroughputBuckets({ + throughputResultBuckets: response.aggregations?.throughput.buckets, + bucketSize, + setupTimeRange: setup, + }), + }; } diff --git a/x-pack/plugins/apm/server/lib/transactions/get_transaction/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_transaction/index.ts index 6987ef07577348..c928b00cefb63c 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_transaction/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_transaction/index.ts @@ -13,9 +13,8 @@ import { rangeQuery } from '../../../../server/utils/queries'; import { Setup, SetupTimeRange } from '../../helpers/setup_request'; import { ProcessorEvent } from '../../../../common/processor_event'; import { asMutableArray } from '../../../../common/utils/as_mutable_array'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getTransaction({ +export async function getTransaction({ transactionId, traceId, setup, @@ -24,27 +23,25 @@ export function getTransaction({ traceId?: string; setup: Setup | (Setup & SetupTimeRange); }) { - return withApmSpan('get_transaction', async () => { - const { apmEventClient } = setup; + const { apmEventClient } = setup; - const resp = await apmEventClient.search({ - apm: { - events: [ProcessorEvent.transaction], - }, - body: { - size: 1, - query: { - bool: { - filter: asMutableArray([ - { term: { [TRANSACTION_ID]: transactionId } }, - ...(traceId ? [{ term: { [TRACE_ID]: traceId } }] : []), - ...('start' in setup ? rangeQuery(setup.start, setup.end) : []), - ]), - }, + const resp = await apmEventClient.search('get_transaction', { + apm: { + events: [ProcessorEvent.transaction], + }, + body: { + size: 1, + query: { + bool: { + filter: asMutableArray([ + { term: { [TRANSACTION_ID]: transactionId } }, + ...(traceId ? [{ term: { [TRACE_ID]: traceId } }] : []), + ...('start' in setup ? rangeQuery(setup.start, setup.end) : []), + ]), }, }, - }); - - return resp.hits.hits[0]?._source; + }, }); + + return resp.hits.hits[0]?._source; } diff --git a/x-pack/plugins/apm/server/lib/transactions/get_transaction_by_trace/index.ts b/x-pack/plugins/apm/server/lib/transactions/get_transaction_by_trace/index.ts index dfdad2f59a848f..568ce16e7aedc2 100644 --- a/x-pack/plugins/apm/server/lib/transactions/get_transaction_by_trace/index.ts +++ b/x-pack/plugins/apm/server/lib/transactions/get_transaction_by_trace/index.ts @@ -11,40 +11,43 @@ import { } from '../../../../common/elasticsearch_fieldnames'; import { Setup } from '../../helpers/setup_request'; import { ProcessorEvent } from '../../../../common/processor_event'; -import { withApmSpan } from '../../../utils/with_apm_span'; -export function getRootTransactionByTraceId(traceId: string, setup: Setup) { - return withApmSpan('get_root_transaction_by_trace_id', async () => { - const { apmEventClient } = setup; +export async function getRootTransactionByTraceId( + traceId: string, + setup: Setup +) { + const { apmEventClient } = setup; - const params = { - apm: { - events: [ProcessorEvent.transaction as const], - }, - body: { - size: 1, - query: { - bool: { - should: [ - { - constant_score: { - filter: { - bool: { - must_not: { exists: { field: PARENT_ID } }, - }, + const params = { + apm: { + events: [ProcessorEvent.transaction as const], + }, + body: { + size: 1, + query: { + bool: { + should: [ + { + constant_score: { + filter: { + bool: { + must_not: { exists: { field: PARENT_ID } }, }, }, }, - ], - filter: [{ term: { [TRACE_ID]: traceId } }], - }, + }, + ], + filter: [{ term: { [TRACE_ID]: traceId } }], }, }, - }; + }, + }; - const resp = await apmEventClient.search(params); - return { - transaction: resp.hits.hits[0]?._source, - }; - }); + const resp = await apmEventClient.search( + 'get_root_transaction_by_trace_id', + params + ); + return { + transaction: resp.hits.hits[0]?._source, + }; } diff --git a/x-pack/plugins/apm/server/utils/test_helpers.tsx b/x-pack/plugins/apm/server/utils/test_helpers.tsx index 6252c33c5994dd..9c63f0140bdf2b 100644 --- a/x-pack/plugins/apm/server/utils/test_helpers.tsx +++ b/x-pack/plugins/apm/server/utils/test_helpers.tsx @@ -109,7 +109,7 @@ export async function inspectSearchParams( } return { - params: spy.mock.calls[0][0], + params: spy.mock.calls[0][1], response, error, spy, diff --git a/x-pack/plugins/apm/server/utils/with_apm_span.ts b/x-pack/plugins/apm/server/utils/with_apm_span.ts index 9762a7213d0a2b..1343970f04a3f3 100644 --- a/x-pack/plugins/apm/server/utils/with_apm_span.ts +++ b/x-pack/plugins/apm/server/utils/with_apm_span.ts @@ -13,7 +13,7 @@ export function withApmSpan( const options = parseSpanOptions(optionsOrName); const optionsWithDefaults = { - type: 'plugin:apm', + ...(options.intercept ? {} : { type: 'plugin:apm' }), ...options, labels: { plugin: 'apm', From 767b67d0522ec00865a7746d3ec37cd5de077821 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Casper=20H=C3=BCbertz?= Date: Mon, 14 Jun 2021 18:49:58 +0200 Subject: [PATCH 09/77] [APM] Change View full trace button fill (#102066) --- .../WaterfallWithSummmary/MaybeViewTraceLink.tsx | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/MaybeViewTraceLink.tsx b/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/MaybeViewTraceLink.tsx index 4017495dd3b5dd..11a0cc1234f428 100644 --- a/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/MaybeViewTraceLink.tsx +++ b/x-pack/plugins/apm/public/components/app/transaction_details/WaterfallWithSummmary/MaybeViewTraceLink.tsx @@ -45,7 +45,7 @@ export const MaybeViewTraceLink = ({ } )} > - + {viewFullTraceButtonLabel} @@ -67,7 +67,7 @@ export const MaybeViewTraceLink = ({ } )} > - + {viewFullTraceButtonLabel} @@ -92,7 +92,9 @@ export const MaybeViewTraceLink = ({ environment={nextEnvironment} latencyAggregationType={latencyAggregationType} > - {viewFullTraceButtonLabel} + + {viewFullTraceButtonLabel} + ); From 4fb0c943a6b5048c6033f5d8d41729aa2eb4f839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ece=20=C3=96zalp?= Date: Mon, 14 Jun 2021 13:31:20 -0400 Subject: [PATCH 10/77] [Security Solution] Refactor rules for modularization by updating bulkCreate and wrapHits methods (#101544) creates bulkCreate and wrapHits factories for the modularization of the detection engine Co-authored-by: Marshall Main --- .../rules_notification_alert_type.ts | 1 - .../schedule_notification_actions.ts | 3 +- .../__mocks__/build_rule_message.mock.ts | 15 + .../signals/bulk_create_factory.ts | 100 ++++++ .../signals/bulk_create_ml_signals.ts | 15 +- .../signals/executors/eql.test.ts | 39 +-- .../detection_engine/signals/executors/eql.ts | 21 +- .../signals/executors/ml.test.ts | 30 +- .../detection_engine/signals/executors/ml.ts | 21 +- .../signals/executors/query.ts | 12 +- .../signals/executors/threat_match.ts | 12 +- .../signals/executors/threshold.test.ts | 20 +- .../signals/executors/threshold.ts | 27 +- .../signals/filter_duplicate_signals.test.ts | 46 +++ .../signals/filter_duplicate_signals.ts | 14 + .../signals/search_after_bulk_create.test.ts | 67 ++-- .../signals/search_after_bulk_create.ts | 22 +- .../signals/signal_rule_alert_type.test.ts | 30 +- .../signals/signal_rule_alert_type.ts | 37 +- .../signals/single_bulk_create.test.ts | 318 ------------------ .../signals/single_bulk_create.ts | 227 ------------- .../threat_mapping/create_threat_signal.ts | 6 +- .../threat_mapping/create_threat_signals.ts | 7 +- .../signals/threat_mapping/types.ts | 8 +- .../signals/threat_mapping/utils.test.ts | 37 ++ .../signals/threat_mapping/utils.ts | 3 + .../bulk_create_threshold_signals.ts | 146 ++++---- .../lib/detection_engine/signals/types.ts | 14 +- .../detection_engine/signals/utils.test.ts | 13 +- .../lib/detection_engine/signals/utils.ts | 28 +- .../signals/wrap_hits_factory.ts | 35 ++ .../security_solution/server/lib/types.ts | 3 + .../security_solution/server/plugin.ts | 2 + 33 files changed, 543 insertions(+), 836 deletions(-) create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/build_rule_message.mock.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts delete mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts create mode 100644 x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts index 6f8dac5b49b31f..a4863e577c6bca 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/rules_notification_alert_type.ts @@ -91,7 +91,6 @@ export const rulesNotificationAlertType = ({ signalsCount, resultsLink, ruleParams, - // @ts-expect-error @elastic/elasticsearch _source is optional signals, }); } diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts index e7db10380eea11..bfb96e97edf117 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/notifications/schedule_notification_actions.ts @@ -8,7 +8,6 @@ import { mapKeys, snakeCase } from 'lodash/fp'; import { AlertInstance } from '../../../../../alerting/server'; import { RuleParams } from '../schemas/rule_schemas'; -import { SignalSource } from '../signals/types'; export type NotificationRuleTypeParams = RuleParams & { name: string; @@ -20,7 +19,7 @@ interface ScheduleNotificationActions { signalsCount: number; resultsLink: string; ruleParams: NotificationRuleTypeParams; - signals: SignalSource[]; + signals: unknown[]; } export const scheduleNotificationActions = ({ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/build_rule_message.mock.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/build_rule_message.mock.ts new file mode 100644 index 00000000000000..f43142d1d0264f --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/__mocks__/build_rule_message.mock.ts @@ -0,0 +1,15 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { buildRuleMessageFactory } from '../rule_messages'; + +export const mockBuildRuleMessage = buildRuleMessageFactory({ + id: 'fake id', + ruleId: 'fake rule id', + index: 'fakeindex', + name: 'fake name', +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts new file mode 100644 index 00000000000000..f518ac2386d0bb --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_factory.ts @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { performance } from 'perf_hooks'; +import { countBy, isEmpty, get } from 'lodash'; +import { ElasticsearchClient, Logger } from 'kibana/server'; +import { BuildRuleMessage } from './rule_messages'; +import { RefreshTypes } from '../types'; +import { BaseHit } from '../../../../common/detection_engine/types'; +import { errorAggregator, makeFloatString } from './utils'; + +export interface GenericBulkCreateResponse { + success: boolean; + bulkCreateDuration: string; + createdItemsCount: number; + createdItems: Array; + errors: string[]; +} + +export const bulkCreateFactory = ( + logger: Logger, + esClient: ElasticsearchClient, + buildRuleMessage: BuildRuleMessage, + refreshForBulkCreate: RefreshTypes +) => async (wrappedDocs: Array>): Promise> => { + if (wrappedDocs.length === 0) { + return { + errors: [], + success: true, + bulkCreateDuration: '0', + createdItemsCount: 0, + createdItems: [], + }; + } + + const bulkBody = wrappedDocs.flatMap((wrappedDoc) => [ + { + create: { + _index: wrappedDoc._index, + _id: wrappedDoc._id, + }, + }, + wrappedDoc._source, + ]); + const start = performance.now(); + + const { body: response } = await esClient.bulk({ + refresh: refreshForBulkCreate, + body: bulkBody, + }); + + const end = performance.now(); + logger.debug( + buildRuleMessage( + `individual bulk process time took: ${makeFloatString(end - start)} milliseconds` + ) + ); + logger.debug(buildRuleMessage(`took property says bulk took: ${response.took} milliseconds`)); + const createdItems = wrappedDocs + .map((doc, index) => ({ + _id: response.items[index].create?._id ?? '', + _index: response.items[index].create?._index ?? '', + ...doc._source, + })) + .filter((_, index) => get(response.items[index], 'create.status') === 201); + const createdItemsCount = createdItems.length; + const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; + const errorCountByMessage = errorAggregator(response, [409]); + + logger.debug(buildRuleMessage(`bulk created ${createdItemsCount} signals`)); + if (duplicateSignalsCount > 0) { + logger.debug(buildRuleMessage(`ignored ${duplicateSignalsCount} duplicate signals`)); + } + if (!isEmpty(errorCountByMessage)) { + logger.error( + buildRuleMessage( + `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` + ) + ); + return { + errors: Object.keys(errorCountByMessage), + success: false, + bulkCreateDuration: makeFloatString(end - start), + createdItemsCount, + createdItems, + }; + } else { + return { + errors: [], + success: true, + bulkCreateDuration: makeFloatString(end - start), + createdItemsCount, + createdItems, + }; + } +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts index 00ac40fa7e27ce..ebb4462817eabc 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/bulk_create_ml_signals.ts @@ -14,11 +14,10 @@ import { AlertInstanceState, AlertServices, } from '../../../../../alerting/server'; -import { RefreshTypes } from '../types'; -import { singleBulkCreate, SingleBulkCreateResponse } from './single_bulk_create'; +import { GenericBulkCreateResponse } from './bulk_create_factory'; import { AnomalyResults, Anomaly } from '../../machine_learning'; import { BuildRuleMessage } from './rule_messages'; -import { AlertAttributes } from './types'; +import { AlertAttributes, BulkCreate, WrapHits } from './types'; import { MachineLearningRuleParams } from '../schemas/rule_schemas'; interface BulkCreateMlSignalsParams { @@ -28,8 +27,9 @@ interface BulkCreateMlSignalsParams { logger: Logger; id: string; signalsIndex: string; - refresh: RefreshTypes; buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } interface EcsAnomaly extends Anomaly { @@ -85,9 +85,10 @@ const transformAnomalyResultsToEcs = ( export const bulkCreateMlSignals = async ( params: BulkCreateMlSignalsParams -): Promise => { +): Promise> => { const anomalyResults = params.someResult; const ecsResults = transformAnomalyResultsToEcs(anomalyResults); - const buildRuleMessage = params.buildRuleMessage; - return singleBulkCreate({ ...params, filteredEvents: ecsResults, buildRuleMessage }); + + const wrappedDocs = params.wrapHits(ecsResults.hits.hits); + return params.bulkCreate(wrappedDocs); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts index f8f77bd2bf6e6d..947e7d573173ea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.test.ts @@ -7,7 +7,6 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; -import { RuleStatusService } from '../rule_status_service'; import { eqlExecutor } from './eql'; import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock'; @@ -23,7 +22,6 @@ describe('eql_executor', () => { const version = '8.0.0'; let logger: ReturnType; let alertServices: AlertServicesMock; - let ruleStatusService: Record; (getIndexVersion as jest.Mock).mockReturnValue(SIGNALS_TEMPLATE_VERSION); const eqlSO = { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', @@ -51,17 +49,11 @@ describe('eql_executor', () => { beforeEach(() => { alertServices = alertsMock.createAlertServices(); logger = loggingSystemMock.createLogger(); - ruleStatusService = { - success: jest.fn(), - find: jest.fn(), - goingToRun: jest.fn(), - error: jest.fn(), - partialFailure: jest.fn(), - }; alertServices.scopedClusterClient.asCurrentUser.transport.request.mockResolvedValue( elasticsearchClientMock.createSuccessTransportRequestPromise({ hits: { total: { value: 10 }, + events: [], }, }) ); @@ -70,25 +62,16 @@ describe('eql_executor', () => { describe('eqlExecutor', () => { it('should set a warning when exception list for eql rule contains value list exceptions', async () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; - try { - await eqlExecutor({ - rule: eqlSO, - exceptionItems, - ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, - services: alertServices, - version, - logger, - refresh: false, - searchAfterSize, - }); - } catch (err) { - // eqlExecutor will throw until we have an EQL response mock that conforms to the - // expected EQL response format, so just catch the error and check the status service - } - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' - ); + const response = await eqlExecutor({ + rule: eqlSO, + exceptionItems, + services: alertServices, + version, + logger, + searchAfterSize, + bulkCreate: jest.fn(), + }); + expect(response.warningMessages.length).toEqual(1); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts index 8e2f5e92ae5028..28d1f3e19baeed 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/eql.ts @@ -7,9 +7,9 @@ import { ApiResponse } from '@elastic/elasticsearch'; import { performance } from 'perf_hooks'; -import { Logger } from 'src/core/server'; import { SavedObject } from 'src/core/types'; import type { ExceptionListItemSchema } from '@kbn/securitysolution-io-ts-list-types'; +import { Logger } from 'src/core/server'; import { AlertInstanceContext, AlertInstanceState, @@ -21,13 +21,12 @@ import { isOutdated } from '../../migrations/helpers'; import { getIndexVersion } from '../../routes/index/get_index_version'; import { MIN_EQL_RULE_INDEX_VERSION } from '../../routes/index/get_signals_template'; import { EqlRuleParams } from '../../schemas/rule_schemas'; -import { RefreshTypes } from '../../types'; import { buildSignalFromEvent, buildSignalGroupFromSequence } from '../build_bulk_body'; import { getInputIndex } from '../get_input_output_index'; -import { RuleStatusService } from '../rule_status_service'; -import { bulkInsertSignals, filterDuplicateSignals } from '../single_bulk_create'; +import { filterDuplicateSignals } from '../filter_duplicate_signals'; import { AlertAttributes, + BulkCreate, EqlSignalSearchResponse, SearchAfterAndBulkCreateReturnType, WrappedSignalHit, @@ -37,26 +36,24 @@ import { createSearchAfterReturnType, makeFloatString, wrapSignal } from '../uti export const eqlExecutor = async ({ rule, exceptionItems, - ruleStatusService, services, version, - searchAfterSize, logger, - refresh, + searchAfterSize, + bulkCreate, }: { rule: SavedObject>; exceptionItems: ExceptionListItemSchema[]; - ruleStatusService: RuleStatusService; services: AlertServices; version: string; - searchAfterSize: number; logger: Logger; - refresh: RefreshTypes; + searchAfterSize: number; + bulkCreate: BulkCreate; }): Promise => { const result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; if (hasLargeValueItem(exceptionItems)) { - await ruleStatusService.partialFailure( + result.warningMessages.push( 'Exceptions that use "is in list" or "is not in list" operators are not applied to EQL rules' ); result.warning = true; @@ -125,7 +122,7 @@ export const eqlExecutor = async ({ } if (newSignals.length > 0) { - const insertResult = await bulkInsertSignals(newSignals, logger, services, refresh); + const insertResult = await bulkCreate(newSignals); result.bulkCreateTimes.push(insertResult.bulkCreateDuration); result.createdSignalsCount += insertResult.createdItemsCount; result.createdSignals = insertResult.createdItems; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts index e157750a7d51bd..25a9d2c3f510fe 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.test.ts @@ -7,7 +7,6 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; -import { RuleStatusService } from '../rule_status_service'; import { mlExecutor } from './ml'; import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getMlRuleParams } from '../../schemas/rule_schemas.mock'; @@ -17,7 +16,6 @@ import { findMlSignals } from '../find_ml_signals'; import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; import { mlPluginServerMock } from '../../../../../../ml/server/mocks'; import { sampleRuleSO } from '../__mocks__/es_results'; -import { getRuleStatusServiceMock } from '../rule_status_service.mock'; jest.mock('../find_ml_signals'); jest.mock('../bulk_create_ml_signals'); @@ -25,7 +23,6 @@ jest.mock('../bulk_create_ml_signals'); describe('ml_executor', () => { let jobsSummaryMock: jest.Mock; let mlMock: ReturnType; - let ruleStatusService: ReturnType; const exceptionItems = [getExceptionListItemSchemaMock()]; let logger: ReturnType; let alertServices: AlertServicesMock; @@ -45,7 +42,6 @@ describe('ml_executor', () => { mlMock.jobServiceProvider.mockReturnValue({ jobsSummary: jobsSummaryMock, }); - ruleStatusService = getRuleStatusServiceMock(); (findMlSignals as jest.Mock).mockResolvedValue({ _shards: {}, hits: { @@ -66,35 +62,32 @@ describe('ml_executor', () => { rule: mlSO, ml: undefined, exceptionItems, - ruleStatusService, services: alertServices, logger, - refresh: false, buildRuleMessage, listClient: getListClientMock(), + bulkCreate: jest.fn(), + wrapHits: jest.fn(), }) ).rejects.toThrow('ML plugin unavailable during rule execution'); }); it('should record a partial failure if Machine learning job summary was null', async () => { jobsSummaryMock.mockResolvedValue([]); - await mlExecutor({ + const response = await mlExecutor({ rule: mlSO, ml: mlMock, exceptionItems, - ruleStatusService, services: alertServices, logger, - refresh: false, buildRuleMessage, listClient: getListClientMock(), + bulkCreate: jest.fn(), + wrapHits: jest.fn(), }); expect(logger.warn).toHaveBeenCalled(); expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job(s) are not started'); - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Machine learning job(s) are not started' - ); + expect(response.warningMessages.length).toEqual(1); }); it('should record a partial failure if Machine learning job was not started', async () => { @@ -106,22 +99,19 @@ describe('ml_executor', () => { }, ]); - await mlExecutor({ + const response = await mlExecutor({ rule: mlSO, ml: mlMock, exceptionItems, - ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, services: alertServices, logger, - refresh: false, buildRuleMessage, listClient: getListClientMock(), + bulkCreate: jest.fn(), + wrapHits: jest.fn(), }); expect(logger.warn).toHaveBeenCalled(); expect(logger.warn.mock.calls[0][0]).toContain('Machine learning job(s) are not started'); - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Machine learning job(s) are not started' - ); + expect(response.warningMessages.length).toEqual(1); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts index 28703046289f5f..f5c7d8822b51f0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/ml.ts @@ -17,13 +17,11 @@ import { ListClient } from '../../../../../../lists/server'; import { isJobStarted } from '../../../../../common/machine_learning/helpers'; import { SetupPlugins } from '../../../../plugin'; import { MachineLearningRuleParams } from '../../schemas/rule_schemas'; -import { RefreshTypes } from '../../types'; import { bulkCreateMlSignals } from '../bulk_create_ml_signals'; import { filterEventsAgainstList } from '../filters/filter_events_against_list'; import { findMlSignals } from '../find_ml_signals'; import { BuildRuleMessage } from '../rule_messages'; -import { RuleStatusService } from '../rule_status_service'; -import { AlertAttributes } from '../types'; +import { AlertAttributes, BulkCreate, WrapHits } from '../types'; import { createErrorsFromShard, createSearchAfterReturnType, mergeReturns } from '../utils'; export const mlExecutor = async ({ @@ -31,21 +29,21 @@ export const mlExecutor = async ({ ml, listClient, exceptionItems, - ruleStatusService, services, logger, - refresh, buildRuleMessage, + bulkCreate, + wrapHits, }: { rule: SavedObject>; ml: SetupPlugins['ml']; listClient: ListClient; exceptionItems: ExceptionListItemSchema[]; - ruleStatusService: RuleStatusService; services: AlertServices; logger: Logger; - refresh: RefreshTypes; buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; }) => { const result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; @@ -67,7 +65,7 @@ export const mlExecutor = async ({ jobSummaries.length < 1 || jobSummaries.some((job) => !isJobStarted(job.jobState, job.datafeedState)) ) { - const errorMessage = buildRuleMessage( + const warningMessage = buildRuleMessage( 'Machine learning job(s) are not started:', ...jobSummaries.map((job) => [ @@ -77,9 +75,9 @@ export const mlExecutor = async ({ ].join(', ') ) ); - logger.warn(errorMessage); + result.warningMessages.push(warningMessage); + logger.warn(warningMessage); result.warning = true; - await ruleStatusService.partialFailure(errorMessage); } const anomalyResults = await findMlSignals({ @@ -120,8 +118,9 @@ export const mlExecutor = async ({ logger, id: rule.id, signalsIndex: ruleParams.outputIndex, - refresh, buildRuleMessage, + bulkCreate, + wrapHits, }); // The legacy ES client does not define failures when it can be present on the structure, hence why I have the & { failures: [] } const shardFailures = diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts index 05e2e3056e99eb..9d76a06afa2755 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/query.ts @@ -14,11 +14,10 @@ import { AlertServices, } from '../../../../../../alerting/server'; import { ListClient } from '../../../../../../lists/server'; -import { RefreshTypes } from '../../types'; import { getFilter } from '../get_filter'; import { getInputIndex } from '../get_input_output_index'; import { searchAfterAndBulkCreate } from '../search_after_bulk_create'; -import { AlertAttributes, RuleRangeTuple } from '../types'; +import { AlertAttributes, RuleRangeTuple, BulkCreate, WrapHits } from '../types'; import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { QueryRuleParams, SavedQueryRuleParams } from '../../schemas/rule_schemas'; @@ -32,9 +31,10 @@ export const queryExecutor = async ({ version, searchAfterSize, logger, - refresh, eventsTelemetry, buildRuleMessage, + bulkCreate, + wrapHits, }: { rule: SavedObject>; tuples: RuleRangeTuple[]; @@ -44,9 +44,10 @@ export const queryExecutor = async ({ version: string; searchAfterSize: number; logger: Logger; - refresh: RefreshTypes; eventsTelemetry: TelemetryEventsSender | undefined; buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; }) => { const ruleParams = rule.attributes.params; const inputIndex = await getInputIndex(services, version, ruleParams.index); @@ -74,7 +75,8 @@ export const queryExecutor = async ({ signalsIndex: ruleParams.outputIndex, filter: esFilter, pageSize: searchAfterSize, - refresh, buildRuleMessage, + bulkCreate, + wrapHits, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts index 10b4ce939ca3ac..078eb8362069cf 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threat_match.ts @@ -14,9 +14,8 @@ import { AlertServices, } from '../../../../../../alerting/server'; import { ListClient } from '../../../../../../lists/server'; -import { RefreshTypes } from '../../types'; import { getInputIndex } from '../get_input_output_index'; -import { RuleRangeTuple, AlertAttributes } from '../types'; +import { RuleRangeTuple, AlertAttributes, BulkCreate, WrapHits } from '../types'; import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { createThreatSignals } from '../threat_mapping/create_threat_signals'; @@ -31,9 +30,10 @@ export const threatMatchExecutor = async ({ version, searchAfterSize, logger, - refresh, eventsTelemetry, buildRuleMessage, + bulkCreate, + wrapHits, }: { rule: SavedObject>; tuples: RuleRangeTuple[]; @@ -43,9 +43,10 @@ export const threatMatchExecutor = async ({ version: string; searchAfterSize: number; logger: Logger; - refresh: RefreshTypes; eventsTelemetry: TelemetryEventsSender | undefined; buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; }) => { const ruleParams = rule.attributes.params; const inputIndex = await getInputIndex(services, version, ruleParams.index); @@ -67,7 +68,6 @@ export const threatMatchExecutor = async ({ outputIndex: ruleParams.outputIndex, ruleSO: rule, searchAfterSize, - refresh, threatFilters: ruleParams.threatFilters ?? [], threatQuery: ruleParams.threatQuery, threatLanguage: ruleParams.threatLanguage, @@ -76,5 +76,7 @@ export const threatMatchExecutor = async ({ threatIndicatorPath: ruleParams.threatIndicatorPath, concurrentSearches: ruleParams.concurrentSearches ?? 1, itemsPerSearch: ruleParams.itemsPerSearch ?? 9000, + bulkCreate, + wrapHits, }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts index 5d62b28b73ae87..f03e8b8a147aea 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.test.ts @@ -7,7 +7,6 @@ import { loggingSystemMock } from 'src/core/server/mocks'; import { alertsMock, AlertServicesMock } from '../../../../../../alerting/server/mocks'; -import { RuleStatusService } from '../rule_status_service'; import { thresholdExecutor } from './threshold'; import { getExceptionListItemSchemaMock } from '../../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; import { getEntryListMock } from '../../../../../../lists/common/schemas/types/entry_list.mock'; @@ -18,7 +17,6 @@ describe('threshold_executor', () => { const version = '8.0.0'; let logger: ReturnType; let alertServices: AlertServicesMock; - let ruleStatusService: Record; const thresholdSO = { id: '04128c15-0d1b-4716-a4c5-46997ac7f3bd', type: 'alert', @@ -50,34 +48,24 @@ describe('threshold_executor', () => { beforeEach(() => { alertServices = alertsMock.createAlertServices(); logger = loggingSystemMock.createLogger(); - ruleStatusService = { - success: jest.fn(), - find: jest.fn(), - goingToRun: jest.fn(), - error: jest.fn(), - partialFailure: jest.fn(), - }; }); describe('thresholdExecutor', () => { it('should set a warning when exception list for threshold rule contains value list exceptions', async () => { const exceptionItems = [getExceptionListItemSchemaMock({ entries: [getEntryListMock()] })]; - await thresholdExecutor({ + const response = await thresholdExecutor({ rule: thresholdSO, tuples: [], exceptionItems, - ruleStatusService: (ruleStatusService as unknown) as RuleStatusService, services: alertServices, version, logger, - refresh: false, buildRuleMessage, startedAt: new Date(), + bulkCreate: jest.fn(), + wrapHits: jest.fn(), }); - expect(ruleStatusService.partialFailure).toHaveBeenCalled(); - expect(ruleStatusService.partialFailure.mock.calls[0][0]).toContain( - 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' - ); + expect(response.warningMessages.length).toEqual(1); }); }); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts index fa0986044e2502..5e23128c9c148a 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/executors/threshold.ts @@ -15,51 +15,55 @@ import { } from '../../../../../../alerting/server'; import { hasLargeValueItem } from '../../../../../common/detection_engine/utils'; import { ThresholdRuleParams } from '../../schemas/rule_schemas'; -import { RefreshTypes } from '../../types'; import { getFilter } from '../get_filter'; import { getInputIndex } from '../get_input_output_index'; -import { BuildRuleMessage } from '../rule_messages'; -import { RuleStatusService } from '../rule_status_service'; import { bulkCreateThresholdSignals, findThresholdSignals, getThresholdBucketFilters, getThresholdSignalHistory, } from '../threshold'; -import { AlertAttributes, RuleRangeTuple, SearchAfterAndBulkCreateReturnType } from '../types'; +import { + AlertAttributes, + BulkCreate, + RuleRangeTuple, + SearchAfterAndBulkCreateReturnType, + WrapHits, +} from '../types'; import { createSearchAfterReturnType, createSearchAfterReturnTypeFromResponse, mergeReturns, } from '../utils'; +import { BuildRuleMessage } from '../rule_messages'; export const thresholdExecutor = async ({ rule, tuples, exceptionItems, - ruleStatusService, services, version, logger, - refresh, buildRuleMessage, startedAt, + bulkCreate, + wrapHits, }: { rule: SavedObject>; tuples: RuleRangeTuple[]; exceptionItems: ExceptionListItemSchema[]; - ruleStatusService: RuleStatusService; services: AlertServices; version: string; logger: Logger; - refresh: RefreshTypes; buildRuleMessage: BuildRuleMessage; startedAt: Date; + bulkCreate: BulkCreate; + wrapHits: WrapHits; }): Promise => { let result = createSearchAfterReturnType(); const ruleParams = rule.attributes.params; if (hasLargeValueItem(exceptionItems)) { - await ruleStatusService.partialFailure( + result.warningMessages.push( 'Exceptions that use "is in list" or "is not in list" operators are not applied to Threshold rules' ); result.warning = true; @@ -126,14 +130,13 @@ export const thresholdExecutor = async ({ filter: esFilter, services, logger, - id: rule.id, inputIndexPattern: inputIndex, signalsIndex: ruleParams.outputIndex, startedAt, from: tuple.from.toDate(), - refresh, thresholdSignalHistory, - buildRuleMessage, + bulkCreate, + wrapHits, }); result = mergeReturns([ diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts new file mode 100644 index 00000000000000..5c4af83c3b03e3 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.test.ts @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { filterDuplicateSignals } from './filter_duplicate_signals'; +import { sampleWrappedSignalHit } from './__mocks__/es_results'; + +const mockRuleId1 = 'aaaaaaaa'; +const mockRuleId2 = 'bbbbbbbb'; +const mockRuleId3 = 'cccccccc'; + +const createWrappedSignalHitWithRuleId = (ruleId: string) => { + const mockSignal = sampleWrappedSignalHit(); + return { + ...mockSignal, + _source: { + ...mockSignal._source, + signal: { + ...mockSignal._source.signal, + ancestors: [ + { + ...mockSignal._source.signal.ancestors[0], + rule: ruleId, + }, + ], + }, + }, + }; +}; +const mockSignals = [ + createWrappedSignalHitWithRuleId(mockRuleId1), + createWrappedSignalHitWithRuleId(mockRuleId2), +]; + +describe('filterDuplicateSignals', () => { + it('filters duplicate signals', () => { + expect(filterDuplicateSignals(mockRuleId1, mockSignals).length).toEqual(1); + }); + + it('does not filter non-duplicate signals', () => { + expect(filterDuplicateSignals(mockRuleId3, mockSignals).length).toEqual(2); + }); +}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts new file mode 100644 index 00000000000000..a648c053062894 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/filter_duplicate_signals.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { WrappedSignalHit } from './types'; + +export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => { + return signals.filter( + (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId) + ); +}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts index 52c887c3ca55af..e4eb7e854f670f 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.test.ts @@ -16,29 +16,28 @@ import { sampleDocWithSortId, } from './__mocks__/es_results'; import { searchAfterAndBulkCreate } from './search_after_bulk_create'; -import { buildRuleMessageFactory } from './rule_messages'; import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; import uuid from 'uuid'; import { listMock } from '../../../../../lists/server/mocks'; import { getExceptionListItemSchemaMock } from '../../../../../lists/common/schemas/response/exception_list_item_schema.mock'; -import { BulkResponse, RuleRangeTuple } from './types'; +import { BulkCreate, BulkResponse, RuleRangeTuple, WrapHits } from './types'; import type { SearchListItemArraySchema } from '@kbn/securitysolution-io-ts-list-types'; import { getSearchListItemResponseMock } from '../../../../../lists/common/schemas/response/search_list_item_schema.mock'; import { getRuleRangeTuples } from './utils'; // eslint-disable-next-line @kbn/eslint/no-restricted-paths import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; import { getQueryRuleParams } from '../schemas/rule_schemas.mock'; +import { bulkCreateFactory } from './bulk_create_factory'; +import { wrapHitsFactory } from './wrap_hits_factory'; +import { mockBuildRuleMessage } from './__mocks__/build_rule_message.mock'; -const buildRuleMessage = buildRuleMessageFactory({ - id: 'fake id', - ruleId: 'fake rule id', - index: 'fakeindex', - name: 'fake name', -}); +const buildRuleMessage = mockBuildRuleMessage; describe('searchAfterAndBulkCreate', () => { let mockService: AlertServicesMock; + let bulkCreate: BulkCreate; + let wrapHits: WrapHits; let inputIndexPattern: string[] = []; let listClient = listMock.getListClient(); const someGuids = Array.from({ length: 13 }).map(() => uuid.v4()); @@ -61,6 +60,13 @@ describe('searchAfterAndBulkCreate', () => { maxSignals: sampleParams.maxSignals, buildRuleMessage, })); + bulkCreate = bulkCreateFactory( + mockLogger, + mockService.scopedClusterClient.asCurrentUser, + buildRuleMessage, + false + ); + wrapHits = wrapHitsFactory({ ruleSO, signalsIndex: DEFAULT_SIGNALS_INDEX }); }); test('should return success with number of searches less than max signals', async () => { @@ -166,6 +172,7 @@ describe('searchAfterAndBulkCreate', () => { }, }, ]; + const { success, createdSignalsCount, lastLookBackDate } = await searchAfterAndBulkCreate({ tuples, ruleSO, @@ -179,8 +186,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(5); @@ -282,8 +290,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(4); @@ -359,8 +368,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); @@ -417,8 +427,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); @@ -495,8 +506,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); @@ -549,8 +561,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1); @@ -625,18 +638,14 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(1); expect(createdSignalsCount).toEqual(4); expect(lastLookBackDate).toEqual(new Date('2020-04-20T21:27:45+0000')); - // I don't like testing log statements since logs change but this is the best - // way I can think of to ensure this section is getting hit with this test case. - expect(((mockLogger.debug as unknown) as jest.Mock).mock.calls[14][0]).toContain( - 'ran out of sort ids to sort on name: "fake name" id: "fake id" rule id: "fake rule id" signals index: "fakeindex"' - ); }); test('should return success when no exceptions list provided', async () => { @@ -703,8 +712,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(mockService.scopedClusterClient.asCurrentUser.search).toHaveBeenCalledTimes(2); @@ -746,8 +756,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(mockLogger.error).toHaveBeenCalled(); expect(success).toEqual(false); @@ -792,8 +803,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(true); expect(createdSignalsCount).toEqual(0); @@ -852,8 +864,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(false); expect(createdSignalsCount).toEqual(0); // should not create signals if search threw error @@ -977,8 +990,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(success).toEqual(false); expect(errors).toEqual(['error on creation']); @@ -1072,8 +1086,9 @@ describe('searchAfterAndBulkCreate', () => { signalsIndex: DEFAULT_SIGNALS_INDEX, pageSize: 1, filter: undefined, - refresh: false, buildRuleMessage, + bulkCreate, + wrapHits, }); expect(mockEnrichment).toHaveBeenCalledWith( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts index b0dcc1810a6396..bb2e57b0606e59 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/search_after_bulk_create.ts @@ -8,7 +8,6 @@ import { identity } from 'lodash'; import type { estypes } from '@elastic/elasticsearch'; import { singleSearchAfter } from './single_search_after'; -import { singleBulkCreate } from './single_bulk_create'; import { filterEventsAgainstList } from './filters/filter_events_against_list'; import { sendAlertTelemetryEvents } from './send_telemetry_events'; import { @@ -31,14 +30,13 @@ export const searchAfterAndBulkCreate = async ({ listClient, logger, eventsTelemetry, - id, inputIndexPattern, - signalsIndex, filter, pageSize, - refresh, buildRuleMessage, enrichment = identity, + bulkCreate, + wrapHits, }: SearchAfterAndBulkCreateParams): Promise => { const ruleParams = ruleSO.attributes.params; let toReturn = createSearchAfterReturnType(); @@ -149,6 +147,7 @@ export const searchAfterAndBulkCreate = async ({ ); } const enrichedEvents = await enrichment(filteredEvents); + const wrappedDocs = wrapHits(enrichedEvents.hits.hits); const { bulkCreateDuration: bulkDuration, @@ -156,16 +155,7 @@ export const searchAfterAndBulkCreate = async ({ createdItems, success: bulkSuccess, errors: bulkErrors, - } = await singleBulkCreate({ - buildRuleMessage, - filteredEvents: enrichedEvents, - ruleSO, - services, - logger, - id, - signalsIndex, - refresh, - }); + } = await bulkCreate(wrappedDocs); toReturn = mergeReturns([ toReturn, createSearchAfterReturnType({ @@ -180,10 +170,10 @@ export const searchAfterAndBulkCreate = async ({ logger.debug(buildRuleMessage(`created ${createdCount} signals`)); logger.debug(buildRuleMessage(`signalsCreatedCount: ${signalsCreatedCount}`)); logger.debug( - buildRuleMessage(`filteredEvents.hits.hits: ${filteredEvents.hits.hits.length}`) + buildRuleMessage(`enrichedEvents.hits.hits: ${enrichedEvents.hits.hits.length}`) ); - sendAlertTelemetryEvents(logger, eventsTelemetry, filteredEvents, buildRuleMessage); + sendAlertTelemetryEvents(logger, eventsTelemetry, enrichedEvents, buildRuleMessage); } if (!hasSortId) { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts index 823d694f36514d..d8c919b50e9db0 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.test.ts @@ -274,35 +274,6 @@ describe('signal_rule_alert_type', () => { expect(ruleStatusService.error).toHaveBeenCalledTimes(0); }); - it("should set refresh to 'wait_for' when actions are present", async () => { - const ruleAlert = getAlertMock(getQueryRuleParams()); - ruleAlert.actions = [ - { - actionTypeId: '.slack', - params: { - message: - 'Rule generated {{state.signals_count}} signals\n\n{{context.rule.name}}\n{{{context.results_link}}}', - }, - group: 'default', - id: '99403909-ca9b-49ba-9d7a-7e5320e68d05', - }, - ]; - - alertServices.savedObjectsClient.get.mockResolvedValue({ - id: 'id', - type: 'type', - references: [], - attributes: ruleAlert, - }); - await alert.executor(payload); - expect((queryExecutor as jest.Mock).mock.calls[0][0].refresh).toEqual('wait_for'); - }); - - it('should set refresh to false when actions are not present', async () => { - await alert.executor(payload); - expect((queryExecutor as jest.Mock).mock.calls[0][0].refresh).toEqual(false); - }); - it('should call scheduleActions if signalsCount was greater than 0 and rule has actions defined', async () => { const ruleAlert = getAlertMock(getQueryRuleParams()); ruleAlert.actions = [ @@ -462,6 +433,7 @@ describe('signal_rule_alert_type', () => { lastLookBackDate: null, createdSignalsCount: 0, createdSignals: [], + warningMessages: [], errors: ['Error that bubbled up.'], }; (queryExecutor as jest.Mock).mockResolvedValue(result); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts index 13a63df6ed8b61..0a2e22bc44b60e 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/signal_rule_alert_type.ts @@ -65,6 +65,8 @@ import { RuleParams, savedQueryRuleParams, } from '../schemas/rule_schemas'; +import { bulkCreateFactory } from './bulk_create_factory'; +import { wrapHitsFactory } from './wrap_hits_factory'; export const signalRulesAlertType = ({ logger, @@ -218,6 +220,19 @@ export const signalRulesAlertType = ({ client: exceptionsClient, lists: params.exceptionsList ?? [], }); + + const bulkCreate = bulkCreateFactory( + logger, + services.scopedClusterClient.asCurrentUser, + buildRuleMessage, + refresh + ); + + const wrapHits = wrapHitsFactory({ + ruleSO: savedObject, + signalsIndex: params.outputIndex, + }); + if (isMlRule(type)) { const mlRuleSO = asTypeSpecificSO(savedObject, machineLearningRuleParams); result = await mlExecutor({ @@ -225,11 +240,11 @@ export const signalRulesAlertType = ({ ml, listClient, exceptionItems, - ruleStatusService, services, logger, - refresh, buildRuleMessage, + bulkCreate, + wrapHits, }); } else if (isThresholdRule(type)) { const thresholdRuleSO = asTypeSpecificSO(savedObject, thresholdRuleParams); @@ -237,13 +252,13 @@ export const signalRulesAlertType = ({ rule: thresholdRuleSO, tuples, exceptionItems, - ruleStatusService, services, version, logger, - refresh, buildRuleMessage, startedAt, + bulkCreate, + wrapHits, }); } else if (isThreatMatchRule(type)) { const threatRuleSO = asTypeSpecificSO(savedObject, threatRuleParams); @@ -256,9 +271,10 @@ export const signalRulesAlertType = ({ version, searchAfterSize, logger, - refresh, eventsTelemetry, buildRuleMessage, + bulkCreate, + wrapHits, }); } else if (isQueryRule(type)) { const queryRuleSO = validateQueryRuleTypes(savedObject); @@ -271,25 +287,30 @@ export const signalRulesAlertType = ({ version, searchAfterSize, logger, - refresh, eventsTelemetry, buildRuleMessage, + bulkCreate, + wrapHits, }); } else if (isEqlRule(type)) { const eqlRuleSO = asTypeSpecificSO(savedObject, eqlRuleParams); result = await eqlExecutor({ rule: eqlRuleSO, exceptionItems, - ruleStatusService, services, version, searchAfterSize, + bulkCreate, logger, - refresh, }); } else { throw new Error(`unknown rule type ${type}`); } + if (result.warningMessages.length) { + const warningMessage = buildRuleMessage(result.warningMessages.join()); + await ruleStatusService.partialFailure(warningMessage); + } + if (result.success) { if (actions.length) { const notificationRuleParams: NotificationRuleTypeParams = { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts deleted file mode 100644 index 3fbb8c1a607e91..00000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.test.ts +++ /dev/null @@ -1,318 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { generateId } from './utils'; -import { - sampleDocSearchResultsNoSortId, - mockLogger, - sampleRuleGuid, - sampleDocSearchResultsNoSortIdNoVersion, - sampleEmptyDocSearchResults, - sampleBulkCreateDuplicateResult, - sampleBulkCreateErrorResult, - sampleDocWithAncestors, - sampleRuleSO, -} from './__mocks__/es_results'; -import { DEFAULT_SIGNALS_INDEX } from '../../../../common/constants'; -import { singleBulkCreate, filterDuplicateRules } from './single_bulk_create'; -import { alertsMock, AlertServicesMock } from '../../../../../alerting/server/mocks'; -import { buildRuleMessageFactory } from './rule_messages'; -// eslint-disable-next-line @kbn/eslint/no-restricted-paths -import { elasticsearchClientMock } from 'src/core/server/elasticsearch/client/mocks'; -import { getQueryRuleParams } from '../schemas/rule_schemas.mock'; - -const buildRuleMessage = buildRuleMessageFactory({ - id: 'fake id', - ruleId: 'fake rule id', - index: 'fakeindex', - name: 'fake name', -}); -describe('singleBulkCreate', () => { - const mockService: AlertServicesMock = alertsMock.createAlertServices(); - - beforeEach(() => { - jest.clearAllMocks(); - }); - - describe('create signal id gereateId', () => { - test('two docs with same index, id, and version should have same id', () => { - const findex = 'myfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const ruleId = 'rule-1'; - // 'myfakeindexsomefakeid1rule-1' - const generatedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex, fid, version, ruleId); - expect(firstHash).toEqual(generatedHash); - expect(secondHash).toEqual(generatedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - }); - test('two docs with different index, id, and version should have different id', () => { - const findex = 'myfakeindex'; - const findex2 = 'mysecondfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const ruleId = 'rule-1'; - // 'myfakeindexsomefakeid1rule-1' - const firstGeneratedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - // 'mysecondfakeindexsomefakeid1rule-1' - const secondGeneratedHash = - 'a852941273f805ffe9006e574601acc8ae1148d6c0b3f7f8c4785cba8f6b768a'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex2, fid, version, ruleId); - expect(firstHash).toEqual(firstGeneratedHash); - expect(secondHash).toEqual(secondGeneratedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - expect(firstHash).not.toEqual(secondHash); - }); - test('two docs with same index, different id, and same version should have different id', () => { - const findex = 'myfakeindex'; - const fid = 'somefakeid'; - const fid2 = 'somefakeid2'; - const version = '1'; - const ruleId = 'rule-1'; - // 'myfakeindexsomefakeid1rule-1' - const firstGeneratedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - // 'myfakeindexsomefakeid21rule-1' - const secondGeneratedHash = - '7d33faea18159fd010c4b79890620e8b12cdc88ec1d370149d0e5552ce860255'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex, fid2, version, ruleId); - expect(firstHash).toEqual(firstGeneratedHash); - expect(secondHash).toEqual(secondGeneratedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - expect(firstHash).not.toEqual(secondHash); - }); - test('two docs with same index, same id, and different version should have different id', () => { - const findex = 'myfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const version2 = '2'; - const ruleId = 'rule-1'; - // 'myfakeindexsomefakeid1rule-1' - const firstGeneratedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - // myfakeindexsomefakeid2rule-1' - const secondGeneratedHash = - 'f016f3071fa9df9221d2fb2ba92389d4d388a4347c6ec7a4012c01cb1c640a40'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex, fid, version2, ruleId); - expect(firstHash).toEqual(firstGeneratedHash); - expect(secondHash).toEqual(secondGeneratedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - expect(firstHash).not.toEqual(secondHash); - }); - test('Ensure generated id is less than 512 bytes, even for really really long strings', () => { - const longIndexName = - 'myfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindexmyfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const ruleId = 'rule-1'; - const firstHash = generateId(longIndexName, fid, version, ruleId); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - }); - test('two docs with same index, same id, same version number, and different rule ids should have different id', () => { - const findex = 'myfakeindex'; - const fid = 'somefakeid'; - const version = '1'; - const ruleId = 'rule-1'; - const ruleId2 = 'rule-2'; - // 'myfakeindexsomefakeid1rule-1' - const firstGeneratedHash = '342404d620be4344d6d90dd0461d1d1848aec457944d5c5f40cc0cbfedb36679'; - // myfakeindexsomefakeid1rule-2' - const secondGeneratedHash = - '1eb04f997086f8b3b143d4d9b18ac178c4a7423f71a5dad9ba8b9e92603c6863'; - const firstHash = generateId(findex, fid, version, ruleId); - const secondHash = generateId(findex, fid, version, ruleId2); - expect(firstHash).toEqual(firstGeneratedHash); - expect(secondHash).toEqual(secondGeneratedHash); - expect(Buffer.byteLength(firstHash, 'utf8')).toBeLessThan(512); // 512 bytes is maximum size of _id field - expect(Buffer.byteLength(secondHash, 'utf8')).toBeLessThan(512); - expect(firstHash).not.toEqual(secondHash); - }); - }); - - test('create successful bulk create', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce( - // @ts-expect-error not compatible response interface - elasticsearchClientMock.createSuccessTransportRequestPromise({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortId(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(0); - }); - - test('create successful bulk create with docs with no versioning', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce( - // @ts-expect-error not compatible response interface - elasticsearchClientMock.createSuccessTransportRequestPromise({ - took: 100, - errors: false, - items: [ - { - fakeItemValue: 'fakeItemKey', - }, - ], - }) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortIdNoVersion(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(0); - }); - - test('create unsuccessful bulk create due to empty search results', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValue( - // @ts-expect-error not full response interface - elasticsearchClientMock.createSuccessTransportRequestPromise(false) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleEmptyDocSearchResults(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(0); - }); - - test('create successful bulk create when bulk create has duplicate errors', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleBulkCreateDuplicateResult) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortId(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - - expect(mockLogger.error).not.toHaveBeenCalled(); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(1); - }); - - test('create failed bulk create when bulk create has multiple error statuses', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValue( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleBulkCreateErrorResult) - ); - const { success, createdItemsCount, errors } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortId(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(mockLogger.error).toHaveBeenCalled(); - expect(errors).toEqual(['[4]: internal server error']); - expect(success).toEqual(false); - expect(createdItemsCount).toEqual(1); - }); - - test('filter duplicate rules will return an empty array given an empty array', () => { - const filtered = filterDuplicateRules( - '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - sampleEmptyDocSearchResults() - ); - expect(filtered).toEqual([]); - }); - - test('filter duplicate rules will return nothing filtered when the two rule ids do not match with each other', () => { - const filtered = filterDuplicateRules('some id', sampleDocWithAncestors()); - expect(filtered).toEqual(sampleDocWithAncestors().hits.hits); - }); - - test('filters duplicate rules will return empty array when the two rule ids match each other', () => { - const filtered = filterDuplicateRules( - '04128c15-0d1b-4716-a4c5-46997ac7f3bd', - sampleDocWithAncestors() - ); - expect(filtered).toEqual([]); - }); - - test('filter duplicate rules will return back search responses if they do not have a signal and will NOT filter the source out', () => { - const ancestors = sampleDocSearchResultsNoSortId(); - const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', ancestors); - expect(filtered).toEqual(ancestors.hits.hits); - }); - - test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own numeric signal type', () => { - const doc = { ...sampleDocWithAncestors(), _source: { signal: 1234 } }; - const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); - expect(filtered).toEqual([]); - }); - - test('filter duplicate rules does not attempt filters when the signal is not an event type of signal but rather a "clash" from the source index having its own object signal type', () => { - const doc = { ...sampleDocWithAncestors(), _source: { signal: {} } }; - const filtered = filterDuplicateRules('04128c15-0d1b-4716-a4c5-46997ac7f3bd', doc); - expect(filtered).toEqual([]); - }); - - test('create successful and returns proper createdItemsCount', async () => { - const ruleSO = sampleRuleSO(getQueryRuleParams()); - mockService.scopedClusterClient.asCurrentUser.bulk.mockResolvedValueOnce( - elasticsearchClientMock.createSuccessTransportRequestPromise(sampleBulkCreateDuplicateResult) - ); - const { success, createdItemsCount } = await singleBulkCreate({ - filteredEvents: sampleDocSearchResultsNoSortId(), - ruleSO, - services: mockService, - logger: mockLogger, - id: sampleRuleGuid, - signalsIndex: DEFAULT_SIGNALS_INDEX, - refresh: false, - buildRuleMessage, - }); - expect(success).toEqual(true); - expect(createdItemsCount).toEqual(1); - }); -}); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts deleted file mode 100644 index 92d01fef6e50c3..00000000000000 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/single_bulk_create.ts +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { countBy, isEmpty, get } from 'lodash'; -import { performance } from 'perf_hooks'; -import { - AlertInstanceContext, - AlertInstanceState, - AlertServices, -} from '../../../../../alerting/server'; -import { AlertAttributes, SignalHit, SignalSearchResponse, WrappedSignalHit } from './types'; -import { RefreshTypes } from '../types'; -import { generateId, makeFloatString, errorAggregator } from './utils'; -import { buildBulkBody } from './build_bulk_body'; -import { BuildRuleMessage } from './rule_messages'; -import { Logger, SavedObject } from '../../../../../../../src/core/server'; -import { isEventTypeSignal } from './build_event_type_signal'; - -interface SingleBulkCreateParams { - filteredEvents: SignalSearchResponse; - ruleSO: SavedObject; - services: AlertServices; - logger: Logger; - id: string; - signalsIndex: string; - refresh: RefreshTypes; - buildRuleMessage: BuildRuleMessage; -} - -/** - * This is for signals on signals to work correctly. If given a rule id this will check if - * that rule id already exists in the ancestor tree of each signal search response and remove - * those documents so they cannot be created as a signal since we do not want a rule id to - * ever be capable of re-writing the same signal continuously if both the _input_ and _output_ - * of the signals index happens to be the same index. - * @param ruleId The rule id - * @param signalSearchResponse The search response that has all the documents - */ -export const filterDuplicateRules = ( - ruleId: string, - signalSearchResponse: SignalSearchResponse -) => { - return signalSearchResponse.hits.hits.filter((doc) => { - if (doc._source?.signal == null || !isEventTypeSignal(doc)) { - return true; - } else { - return !( - doc._source?.signal.ancestors.some((ancestor) => ancestor.rule === ruleId) || - doc._source?.signal.rule.id === ruleId - ); - } - }); -}; - -/** - * Similar to filterDuplicateRules, but operates on candidate signal documents rather than events that matched - * the detection query. This means we only have to compare the ruleId against the ancestors array. - * @param ruleId The rule id - * @param signals The candidate new signals - */ -export const filterDuplicateSignals = (ruleId: string, signals: WrappedSignalHit[]) => { - return signals.filter( - (doc) => !doc._source.signal?.ancestors.some((ancestor) => ancestor.rule === ruleId) - ); -}; - -export interface SingleBulkCreateResponse { - success: boolean; - bulkCreateDuration?: string; - createdItemsCount: number; - createdItems: SignalHit[]; - errors: string[]; -} - -export interface BulkInsertSignalsResponse { - bulkCreateDuration: string; - createdItemsCount: number; - createdItems: SignalHit[]; -} - -// Bulk Index documents. -export const singleBulkCreate = async ({ - buildRuleMessage, - filteredEvents, - ruleSO, - services, - logger, - id, - signalsIndex, - refresh, -}: SingleBulkCreateParams): Promise => { - const ruleParams = ruleSO.attributes.params; - filteredEvents.hits.hits = filterDuplicateRules(id, filteredEvents); - logger.debug(buildRuleMessage(`about to bulk create ${filteredEvents.hits.hits.length} events`)); - if (filteredEvents.hits.hits.length === 0) { - logger.debug(buildRuleMessage(`all events were duplicates`)); - return { success: true, createdItemsCount: 0, createdItems: [], errors: [] }; - } - // index documents after creating an ID based on the - // source documents' originating index, and the original - // document _id. This will allow two documents from two - // different indexes with the same ID to be - // indexed, and prevents us from creating any updates - // to the documents once inserted into the signals index, - // while preventing duplicates from being added to the - // signals index if rules are re-run over the same time - // span. Also allow for versioning. - const bulkBody = filteredEvents.hits.hits.flatMap((doc) => [ - { - create: { - _index: signalsIndex, - _id: generateId( - doc._index, - doc._id, - doc._version ? doc._version.toString() : '', - ruleParams.ruleId ?? '' - ), - }, - }, - buildBulkBody(ruleSO, doc), - ]); - const start = performance.now(); - const { body: response } = await services.scopedClusterClient.asCurrentUser.bulk({ - index: signalsIndex, - refresh, - body: bulkBody, - }); - const end = performance.now(); - logger.debug( - buildRuleMessage( - `individual bulk process time took: ${makeFloatString(end - start)} milliseconds` - ) - ); - logger.debug(buildRuleMessage(`took property says bulk took: ${response.took} milliseconds`)); - const createdItems = filteredEvents.hits.hits - .map((doc, index) => ({ - _id: response.items[index].create?._id ?? '', - _index: response.items[index].create?._index ?? '', - ...buildBulkBody(ruleSO, doc), - })) - .filter((_, index) => get(response.items[index], 'create.status') === 201); - const createdItemsCount = createdItems.length; - const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; - const errorCountByMessage = errorAggregator(response, [409]); - - logger.debug(buildRuleMessage(`bulk created ${createdItemsCount} signals`)); - if (duplicateSignalsCount > 0) { - logger.debug(buildRuleMessage(`ignored ${duplicateSignalsCount} duplicate signals`)); - } - - if (!isEmpty(errorCountByMessage)) { - logger.error( - buildRuleMessage( - `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` - ) - ); - return { - errors: Object.keys(errorCountByMessage), - success: false, - bulkCreateDuration: makeFloatString(end - start), - createdItemsCount, - createdItems, - }; - } else { - return { - errors: [], - success: true, - bulkCreateDuration: makeFloatString(end - start), - createdItemsCount, - createdItems, - }; - } -}; - -// Bulk Index new signals. -export const bulkInsertSignals = async ( - signals: WrappedSignalHit[], - logger: Logger, - services: AlertServices, - refresh: RefreshTypes -): Promise => { - // index documents after creating an ID based on the - // id and index of each parent and the rule ID - const bulkBody = signals.flatMap((doc) => [ - { - create: { - _index: doc._index, - _id: doc._id, - }, - }, - doc._source, - ]); - const start = performance.now(); - const { body: response } = await services.scopedClusterClient.asCurrentUser.bulk({ - refresh, - body: bulkBody, - }); - const end = performance.now(); - logger.debug(`individual bulk process time took: ${makeFloatString(end - start)} milliseconds`); - logger.debug(`took property says bulk took: ${response.took} milliseconds`); - - if (response.errors) { - const duplicateSignalsCount = countBy(response.items, 'create.status')['409']; - logger.debug(`ignored ${duplicateSignalsCount} duplicate signals`); - const errorCountByMessage = errorAggregator(response, [409]); - if (!isEmpty(errorCountByMessage)) { - logger.error( - `[-] bulkResponse had errors with responses of: ${JSON.stringify(errorCountByMessage)}` - ); - } - } - - const createdItemsCount = countBy(response.items, 'create.status')['201'] ?? 0; - const createdItems = signals - .map((doc, index) => ({ - ...doc._source, - _id: response.items[index].create?._id ?? '', - _index: response.items[index].create?._index ?? '', - })) - .filter((_, index) => get(response.items[index], 'create.status') === 201); - logger.debug(`bulk created ${createdItemsCount} signals`); - return { bulkCreateDuration: makeFloatString(end - start), createdItems, createdItemsCount }; -}; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts index 37b0b88d88edab..3e30a08f1ae69c 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signal.ts @@ -31,10 +31,11 @@ export const createThreatSignal = async ({ outputIndex, ruleSO, searchAfterSize, - refresh, buildRuleMessage, currentThreatList, currentResult, + bulkCreate, + wrapHits, }: CreateThreatSignalOptions): Promise => { const threatFilter = buildThreatMappingFilter({ threatMapping, @@ -81,9 +82,10 @@ export const createThreatSignal = async ({ signalsIndex: outputIndex, filter: esFilter, pageSize: searchAfterSize, - refresh, buildRuleMessage, enrichment: threatEnrichment, + bulkCreate, + wrapHits, }); logger.debug( buildRuleMessage( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts index b3e0e376c7794a..5054ab1b2cca50 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/create_threat_signals.ts @@ -32,7 +32,6 @@ export const createThreatSignals = async ({ outputIndex, ruleSO, searchAfterSize, - refresh, threatFilters, threatQuery, threatLanguage, @@ -41,6 +40,8 @@ export const createThreatSignals = async ({ threatIndicatorPath, concurrentSearches, itemsPerSearch, + bulkCreate, + wrapHits, }: CreateThreatSignalsOptions): Promise => { const params = ruleSO.attributes.params; logger.debug(buildRuleMessage('Indicator matching rule starting')); @@ -55,6 +56,7 @@ export const createThreatSignals = async ({ createdSignalsCount: 0, createdSignals: [], errors: [], + warningMessages: [], }; let threatListCount = await getThreatListCount({ @@ -120,10 +122,11 @@ export const createThreatSignals = async ({ outputIndex, ruleSO, searchAfterSize, - refresh, buildRuleMessage, currentThreatList: slicedChunk, currentResult: results, + bulkCreate, + wrapHits, }) ); const searchesPerformed = await Promise.all(concurrentSearchesPerformed); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts index acb64f826f3f24..34b064b0f88053 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/types.ts @@ -29,9 +29,11 @@ import { TelemetryEventsSender } from '../../../telemetry/sender'; import { BuildRuleMessage } from '../rule_messages'; import { AlertAttributes, + BulkCreate, RuleRangeTuple, SearchAfterAndBulkCreateReturnType, SignalsEnrichment, + WrapHits, } from '../types'; import { ThreatRuleParams } from '../../schemas/rule_schemas'; @@ -55,7 +57,6 @@ export interface CreateThreatSignalsOptions { outputIndex: string; ruleSO: SavedObject>; searchAfterSize: number; - refresh: false | 'wait_for'; threatFilters: unknown[]; threatQuery: ThreatQuery; buildRuleMessage: BuildRuleMessage; @@ -64,6 +65,8 @@ export interface CreateThreatSignalsOptions { threatLanguage: ThreatLanguageOrUndefined; concurrentSearches: ConcurrentSearches; itemsPerSearch: ItemsPerSearch; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } export interface CreateThreatSignalOptions { @@ -85,10 +88,11 @@ export interface CreateThreatSignalOptions { outputIndex: string; ruleSO: SavedObject>; searchAfterSize: number; - refresh: false | 'wait_for'; buildRuleMessage: BuildRuleMessage; currentThreatList: ThreatListItem[]; currentResult: SearchAfterAndBulkCreateReturnType; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } export interface BuildThreatMappingFilterOptions { diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts index 6c6447bad09750..ec826b44023f63 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.test.ts @@ -58,6 +58,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -69,6 +70,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults.success).toEqual(true); @@ -84,6 +86,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -95,6 +98,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults.success).toEqual(false); @@ -110,6 +114,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -121,6 +126,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); @@ -136,6 +142,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -147,6 +154,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults).toEqual( @@ -167,6 +175,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 1', 'error 2', 'error 3'], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -178,6 +187,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 4', 'error 1', 'error 3', 'error 5'], + warningMessages: [], }; const combinedResults = combineResults(existingResult, newResult); expect(combinedResults).toEqual( @@ -289,6 +299,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, @@ -299,6 +310,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, []); expect(combinedResults).toEqual(expectedResult); @@ -314,6 +326,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { success: true, @@ -324,6 +337,7 @@ describe('utils', () => { createdSignalsCount: 0, createdSignals: [], errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { success: true, @@ -334,6 +348,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); @@ -350,6 +365,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, @@ -360,6 +376,7 @@ describe('utils', () => { createdSignalsCount: 5, createdSignals: Array(5).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, @@ -370,6 +387,7 @@ describe('utils', () => { createdSignalsCount: 8, createdSignals: Array(8).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { @@ -381,6 +399,7 @@ describe('utils', () => { createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) createdSignals: Array(16).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); @@ -397,6 +416,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, @@ -407,6 +427,7 @@ describe('utils', () => { createdSignalsCount: 5, createdSignals: Array(5).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, @@ -417,6 +438,7 @@ describe('utils', () => { createdSignalsCount: 8, createdSignals: Array(8).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { @@ -428,6 +450,7 @@ describe('utils', () => { createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) createdSignals: Array(16).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult2, newResult1]); // two array elements are flipped @@ -444,6 +467,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult1: SearchAfterAndBulkCreateReturnType = { success: true, @@ -454,6 +478,7 @@ describe('utils', () => { createdSignalsCount: 5, createdSignals: Array(5).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult2: SearchAfterAndBulkCreateReturnType = { success: true, @@ -464,6 +489,7 @@ describe('utils', () => { createdSignalsCount: 8, createdSignals: Array(8).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const expectedResult: SearchAfterAndBulkCreateReturnType = { @@ -475,6 +501,7 @@ describe('utils', () => { createdSignalsCount: 16, // all the signals counted together (8 + 5 + 3) createdSignals: Array(16).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult1, newResult2]); @@ -491,6 +518,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -502,6 +530,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults.success).toEqual(true); @@ -517,6 +546,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -528,6 +558,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults.success).toEqual(false); @@ -543,6 +574,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -554,6 +586,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults.lastLookBackDate?.toISOString()).toEqual('2020-09-16T03:34:32.390Z'); @@ -569,6 +602,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -580,6 +614,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: [], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults).toEqual( @@ -600,6 +635,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 1', 'error 2', 'error 3'], + warningMessages: [], }; const newResult: SearchAfterAndBulkCreateReturnType = { @@ -611,6 +647,7 @@ describe('utils', () => { createdSignalsCount: 3, createdSignals: Array(3).fill(sampleSignalHit()), errors: ['error 4', 'error 1', 'error 3', 'error 5'], + warningMessages: [], }; const combinedResults = combineConcurrentResults(existingResult, [newResult]); expect(combinedResults).toEqual( diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts index 47a32915dd83f4..4d9fda43f032e4 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threat_mapping/utils.ts @@ -75,6 +75,7 @@ export const combineResults = ( lastLookBackDate: newResult.lastLookBackDate, createdSignalsCount: currentResult.createdSignalsCount + newResult.createdSignalsCount, createdSignals: [...currentResult.createdSignals, ...newResult.createdSignals], + warningMessages: [...currentResult.warningMessages, ...newResult.warningMessages], errors: [...new Set([...currentResult.errors, ...newResult.errors])], }); @@ -100,6 +101,7 @@ export const combineConcurrentResults = ( lastLookBackDate, createdSignalsCount: accum.createdSignalsCount + item.createdSignalsCount, createdSignals: [...accum.createdSignals, ...item.createdSignals], + warningMessages: [...accum.warningMessages, ...item.warningMessages], errors: [...new Set([...accum.errors, ...item.errors])], }; }, @@ -112,6 +114,7 @@ export const combineConcurrentResults = ( createdSignalsCount: 0, createdSignals: [], errors: [], + warningMessages: [], } ); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts index 197065f205fc5a..08fa2f14a0fd5d 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/threshold/bulk_create_threshold_signals.ts @@ -19,20 +19,20 @@ import { } from '../../../../../../alerting/server'; import { BaseHit } from '../../../../../common/detection_engine/types'; import { TermAggregationBucket } from '../../../types'; -import { RefreshTypes } from '../../types'; -import { singleBulkCreate, SingleBulkCreateResponse } from '../single_bulk_create'; +import { GenericBulkCreateResponse } from '../bulk_create_factory'; import { calculateThresholdSignalUuid, getThresholdAggregationParts, getThresholdTermsHash, } from '../utils'; -import { BuildRuleMessage } from '../rule_messages'; import type { MultiAggBucket, SignalSource, SignalSearchResponse, ThresholdSignalHistory, AlertAttributes, + BulkCreate, + WrapHits, } from '../types'; import { ThresholdRuleParams } from '../../schemas/rule_schemas'; @@ -42,14 +42,13 @@ interface BulkCreateThresholdSignalsParams { services: AlertServices; inputIndexPattern: string[]; logger: Logger; - id: string; filter: unknown; signalsIndex: string; - refresh: RefreshTypes; startedAt: Date; from: Date; thresholdSignalHistory: ThresholdSignalHistory; - buildRuleMessage: BuildRuleMessage; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } const getTransformedHits = ( @@ -76,7 +75,7 @@ const getTransformedHits = ( return []; } - const getCombinations = (buckets: TermAggregationBucket[], i: number, field: string) => { + const getCombinations = (buckets: TermAggregationBucket[], i: number, field: string | null) => { return buckets.reduce((acc: MultiAggBucket[], bucket: TermAggregationBucket) => { if (i < threshold.field.length - 1) { const nextLevelIdx = i + 1; @@ -100,7 +99,7 @@ const getTransformedHits = ( topThresholdHits: val.topThresholdHits, docCount: val.docCount, }; - acc.push(el); + acc.push(el as MultiAggBucket); }); } else { const el = { @@ -121,80 +120,76 @@ const getTransformedHits = ( topThresholdHits: bucket.top_threshold_hits, docCount: bucket.doc_count, }; - acc.push(el); + acc.push(el as MultiAggBucket); } return acc; }, []); }; - // Recurse through the nested buckets and collect each unique combination of terms. Collect the - // cardinality and document count from the leaf buckets and return a signal for each set of terms. - // @ts-expect-error @elastic/elasticsearch no way to declare a type for aggregation in the search response - return getCombinations(results.aggregations![aggParts.name].buckets, 0, aggParts.field).reduce( - (acc: Array>, bucket) => { - const hit = bucket.topThresholdHits?.hits.hits[0]; - if (hit == null) { - return acc; - } - - const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields); - if (timestampArray == null) { - return acc; - } - - const timestamp = timestampArray[0]; - if (typeof timestamp !== 'string') { - return acc; - } - - const termsHash = getThresholdTermsHash(bucket.terms); - const signalHit = thresholdSignalHistory[termsHash]; - - const source = { - '@timestamp': timestamp, - ...bucket.terms.reduce((termAcc, term) => { - if (!term.field.startsWith('signal.')) { - return { - ...termAcc, - [term.field]: term.value, - }; - } - return termAcc; - }, {}), - threshold_result: { - terms: bucket.terms, - cardinality: bucket.cardinality, - count: bucket.docCount, - // Store `from` in the signal so that we know the lower bound for the - // threshold set in the timeline search. The upper bound will always be - // the `original_time` of the signal (the timestamp of the latest event - // in the set). - from: - signalHit?.lastSignalTimestamp != null - ? new Date(signalHit!.lastSignalTimestamp) - : from, - }, - }; + return getCombinations( + (results.aggregations![aggParts.name] as { buckets: TermAggregationBucket[] }).buckets, + 0, + aggParts.field + ).reduce((acc: Array>, bucket) => { + const hit = bucket.topThresholdHits?.hits.hits[0]; + if (hit == null) { + return acc; + } - acc.push({ - _index: inputIndex, - _id: calculateThresholdSignalUuid( - ruleId, - startedAt, - threshold.field, - bucket.terms - .map((term) => term.value) - .sort() - .join(',') - ), - _source: source, - }); + const timestampArray = get(timestampOverride ?? '@timestamp', hit.fields); + if (timestampArray == null) { + return acc; + } + const timestamp = timestampArray[0]; + if (typeof timestamp !== 'string') { return acc; - }, - [] - ); + } + + const termsHash = getThresholdTermsHash(bucket.terms); + const signalHit = thresholdSignalHistory[termsHash]; + + const source = { + '@timestamp': timestamp, + ...bucket.terms.reduce((termAcc, term) => { + if (!term.field.startsWith('signal.')) { + return { + ...termAcc, + [term.field]: term.value, + }; + } + return termAcc; + }, {}), + threshold_result: { + terms: bucket.terms, + cardinality: bucket.cardinality, + count: bucket.docCount, + // Store `from` in the signal so that we know the lower bound for the + // threshold set in the timeline search. The upper bound will always be + // the `original_time` of the signal (the timestamp of the latest event + // in the set). + from: + signalHit?.lastSignalTimestamp != null ? new Date(signalHit!.lastSignalTimestamp) : from, + }, + }; + + acc.push({ + _index: inputIndex, + _id: calculateThresholdSignalUuid( + ruleId, + startedAt, + threshold.field, + bucket.terms + .map((term) => term.value) + .sort() + .join(',') + ), + _source: source, + }); + + return acc; + }, []); }; export const transformThresholdResultsToEcs = ( @@ -238,7 +233,7 @@ export const transformThresholdResultsToEcs = ( export const bulkCreateThresholdSignals = async ( params: BulkCreateThresholdSignalsParams -): Promise => { +): Promise> => { const ruleParams = params.ruleSO.attributes.params; const thresholdResults = params.someResult; const ecsResults = transformThresholdResultsToEcs( @@ -253,7 +248,6 @@ export const bulkCreateThresholdSignals = async ( ruleParams.timestampOverride, params.thresholdSignalHistory ); - const buildRuleMessage = params.buildRuleMessage; - return singleBulkCreate({ ...params, filteredEvents: ecsResults, buildRuleMessage }); + return params.bulkCreate(params.wrapHits(ecsResults.hits.hits)); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts index 4205c2d6d8b2c5..c35eb04ba12707 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/types.ts @@ -26,12 +26,12 @@ import { RuleAlertAction, SearchTypes, } from '../../../../common/detection_engine/types'; -import { RefreshTypes } from '../types'; import { ListClient } from '../../../../../lists/server'; import { Logger, SavedObject } from '../../../../../../../src/core/server'; import { BuildRuleMessage } from './rule_messages'; import { TelemetryEventsSender } from '../../telemetry/sender'; import { RuleParams } from '../schemas/rule_schemas'; +import { GenericBulkCreateResponse } from './bulk_create_factory'; // used for gap detection code // eslint-disable-next-line @typescript-eslint/naming-convention @@ -255,6 +255,12 @@ export interface QueryFilter { export type SignalsEnrichment = (signals: SignalSearchResponse) => Promise; +export type BulkCreate = (docs: Array>) => Promise>; + +export type WrapHits = ( + hits: Array> +) => Array>; + export interface SearchAfterAndBulkCreateParams { tuples: Array<{ to: moment.Moment; @@ -272,9 +278,10 @@ export interface SearchAfterAndBulkCreateParams { signalsIndex: string; pageSize: number; filter: unknown; - refresh: RefreshTypes; buildRuleMessage: BuildRuleMessage; enrichment?: SignalsEnrichment; + bulkCreate: BulkCreate; + wrapHits: WrapHits; } export interface SearchAfterAndBulkCreateReturnType { @@ -284,8 +291,9 @@ export interface SearchAfterAndBulkCreateReturnType { bulkCreateTimes: string[]; lastLookBackDate: Date | null | undefined; createdSignalsCount: number; - createdSignals: SignalHit[]; + createdSignals: unknown[]; errors: string[]; + warningMessages: string[]; totalToFromTuples?: Array<{ to: Moment | undefined; from: Moment | undefined; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts index 60bf0ec337f3db..616cf714d6a8c2 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.test.ts @@ -1083,6 +1083,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(newSearchResult).toEqual(expected); }); @@ -1102,6 +1103,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(newSearchResult).toEqual(expected); }); @@ -1380,6 +1382,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1394,6 +1397,7 @@ describe('utils', () => { searchAfterTimes: ['123'], success: false, warning: true, + warningMessages: ['test warning'], }); const expected: SearchAfterAndBulkCreateReturnType = { bulkCreateTimes: ['123'], @@ -1404,6 +1408,7 @@ describe('utils', () => { searchAfterTimes: ['123'], success: false, warning: true, + warningMessages: ['test warning'], }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1423,6 +1428,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(searchAfterReturnType).toEqual(expected); }); @@ -1440,6 +1446,7 @@ describe('utils', () => { searchAfterTimes: [], success: true, warning: false, + warningMessages: [], }; expect(merged).toEqual(expected); }); @@ -1494,6 +1501,7 @@ describe('utils', () => { lastLookBackDate: new Date('2020-08-21T18:51:25.193Z'), searchAfterTimes: ['123'], success: true, + warningMessages: ['warning1'], }), createSearchAfterReturnType({ bulkCreateTimes: ['456'], @@ -1503,6 +1511,8 @@ describe('utils', () => { lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), searchAfterTimes: ['567'], success: true, + warningMessages: ['warning2'], + warning: true, }), ]); const expected: SearchAfterAndBulkCreateReturnType = { @@ -1513,7 +1523,8 @@ describe('utils', () => { lastLookBackDate: new Date('2020-09-21T18:51:25.193Z'), // takes the next lastLookBackDate searchAfterTimes: ['123', '567'], // concatenates the searchAfterTimes together success: true, // Defaults to success true is all of it was successful - warning: false, + warning: true, + warningMessages: ['warning1', 'warning2'], }; expect(merged).toEqual(expected); }); diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts index 1de76f64fabec0..6d67bab6eb2f76 100644 --- a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/utils.ts @@ -4,7 +4,6 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ - import { createHash } from 'crypto'; import moment from 'moment'; import uuidv5 from 'uuid/v5'; @@ -17,6 +16,7 @@ import type { ListArray, ExceptionListItemSchema } from '@kbn/securitysolution-i import { MAX_EXCEPTION_LIST_SIZE } from '@kbn/securitysolution-list-constants'; import { hasLargeValueList } from '@kbn/securitysolution-list-utils'; import { parseScheduleDates } from '@kbn/securitysolution-io-ts-utils'; +import { ElasticsearchClient } from '@kbn/securitysolution-es-utils'; import { TimestampOverrideOrUndefined, Privilege, @@ -38,6 +38,7 @@ import { WrappedSignalHit, RuleRangeTuple, BaseSignalHit, + SignalSourceHit, } from './types'; import { BuildRuleMessage } from './rule_messages'; import { ShardError } from '../../types'; @@ -163,9 +164,15 @@ export const hasTimestampFields = async ( export const checkPrivileges = async ( services: AlertServices, indices: string[] +): Promise => + checkPrivilegesFromEsClient(services.scopedClusterClient.asCurrentUser, indices); + +export const checkPrivilegesFromEsClient = async ( + esClient: ElasticsearchClient, + indices: string[] ): Promise => ( - await services.scopedClusterClient.asCurrentUser.transport.request({ + await esClient.transport.request({ path: '/_security/user/_has_privileges', method: 'POST', body: { @@ -608,7 +615,7 @@ export const getValidDateFromDoc = ({ doc.fields != null && doc.fields[timestamp] != null ? doc.fields[timestamp][0] : doc._source != null - ? doc._source[timestamp] + ? (doc._source as { [key: string]: unknown })[timestamp] : undefined; const lastTimestamp = typeof timestampValue === 'string' || typeof timestampValue === 'number' @@ -657,6 +664,7 @@ export const createSearchAfterReturnType = ({ createdSignalsCount, createdSignals, errors, + warningMessages, }: { success?: boolean | undefined; warning?: boolean; @@ -664,8 +672,9 @@ export const createSearchAfterReturnType = ({ bulkCreateTimes?: string[] | undefined; lastLookBackDate?: Date | undefined; createdSignalsCount?: number | undefined; - createdSignals?: SignalHit[] | undefined; + createdSignals?: unknown[] | undefined; errors?: string[] | undefined; + warningMessages?: string[] | undefined; } = {}): SearchAfterAndBulkCreateReturnType => { return { success: success ?? true, @@ -676,10 +685,12 @@ export const createSearchAfterReturnType = ({ createdSignalsCount: createdSignalsCount ?? 0, createdSignals: createdSignals ?? [], errors: errors ?? [], + warningMessages: warningMessages ?? [], }; }; export const createSearchResultReturnType = (): SignalSearchResponse => { + const hits: SignalSourceHit[] = []; return { took: 0, timed_out: false, @@ -693,7 +704,7 @@ export const createSearchResultReturnType = (): SignalSearchResponse => { hits: { total: 0, max_score: 0, - hits: [], + hits, }, }; }; @@ -711,7 +722,8 @@ export const mergeReturns = ( createdSignalsCount: existingCreatedSignalsCount, createdSignals: existingCreatedSignals, errors: existingErrors, - } = prev; + warningMessages: existingWarningMessages, + }: SearchAfterAndBulkCreateReturnType = prev; const { success: newSuccess, @@ -722,7 +734,8 @@ export const mergeReturns = ( createdSignalsCount: newCreatedSignalsCount, createdSignals: newCreatedSignals, errors: newErrors, - } = next; + warningMessages: newWarningMessages, + }: SearchAfterAndBulkCreateReturnType = next; return { success: existingSuccess && newSuccess, @@ -733,6 +746,7 @@ export const mergeReturns = ( createdSignalsCount: existingCreatedSignalsCount + newCreatedSignalsCount, createdSignals: [...existingCreatedSignals, ...newCreatedSignals], errors: [...new Set([...existingErrors, ...newErrors])], + warningMessages: [...existingWarningMessages, ...newWarningMessages], }; }); }; diff --git a/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts new file mode 100644 index 00000000000000..3f3e4ef3631bd9 --- /dev/null +++ b/x-pack/plugins/security_solution/server/lib/detection_engine/signals/wrap_hits_factory.ts @@ -0,0 +1,35 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import { + SearchAfterAndBulkCreateParams, + SignalSourceHit, + WrapHits, + WrappedSignalHit, +} from './types'; +import { generateId } from './utils'; +import { buildBulkBody } from './build_bulk_body'; +import { filterDuplicateSignals } from './filter_duplicate_signals'; + +export const wrapHitsFactory = ({ + ruleSO, + signalsIndex, +}: { + ruleSO: SearchAfterAndBulkCreateParams['ruleSO']; + signalsIndex: string; +}): WrapHits => (events) => { + const wrappedDocs: WrappedSignalHit[] = events.flatMap((doc) => [ + { + _index: signalsIndex, + // TODO: bring back doc._version + _id: generateId(doc._index, doc._id, '', ruleSO.attributes.params.ruleId ?? ''), + _source: buildBulkBody(ruleSO, doc as SignalSourceHit), + }, + ]); + + return filterDuplicateSignals(ruleSO.id, wrappedDocs); +}; diff --git a/x-pack/plugins/security_solution/server/lib/types.ts b/x-pack/plugins/security_solution/server/lib/types.ts index f1c7a275e162c1..6ef51bc3c53d48 100644 --- a/x-pack/plugins/security_solution/server/lib/types.ts +++ b/x-pack/plugins/security_solution/server/lib/types.ts @@ -17,6 +17,7 @@ import { Notes } from './timeline/saved_object/notes'; import { PinnedEvent } from './timeline/saved_object/pinned_events'; import { Timeline } from './timeline/saved_object/timelines'; import { TotalValue, BaseHit, Explanation } from '../../common/detection_engine/types'; +import { SignalHit } from './detection_engine/signals/types'; export interface AppDomainLibs { fields: IndexFields; @@ -100,6 +101,8 @@ export interface SearchResponse extends BaseSearchResponse { export type SearchHit = SearchResponse['hits']['hits'][0]; +export type SearchSignalHit = SearchResponse['hits']['hits'][0]; + export interface TermAggregationBucket { key: string; doc_count: number; diff --git a/x-pack/plugins/security_solution/server/plugin.ts b/x-pack/plugins/security_solution/server/plugin.ts index 609ee88c319f9b..a0f466512cc1d0 100644 --- a/x-pack/plugins/security_solution/server/plugin.ts +++ b/x-pack/plugins/security_solution/server/plugin.ts @@ -260,6 +260,8 @@ export class Plugin implements IPlugin Date: Mon, 14 Jun 2021 20:25:04 +0200 Subject: [PATCH 11/77] [Discover] Unskip runtime field editor test (#101059) --- .../apps/discover/_runtime_fields_editor.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/functional/apps/discover/_runtime_fields_editor.ts b/test/functional/apps/discover/_runtime_fields_editor.ts index 648fa3efe337c0..46fe5c34f4cf36 100644 --- a/test/functional/apps/discover/_runtime_fields_editor.ts +++ b/test/functional/apps/discover/_runtime_fields_editor.ts @@ -31,7 +31,7 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await fieldEditor.save(); }; - describe.skip('discover integration with runtime fields editor', function describeIndexTests() { + describe('discover integration with runtime fields editor', function describeIndexTests() { before(async function () { await esArchiver.load('test/functional/fixtures/es_archiver/discover'); await esArchiver.loadIfNeeded('test/functional/fixtures/es_archiver/logstash_functional'); @@ -104,7 +104,6 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { }); }); - // flaky https://github.com/elastic/kibana/issues/100966 it('doc view includes runtime fields', async function () { // navigate to doc view const table = await PageObjects.discover.getDocTable(); @@ -121,10 +120,16 @@ export default function ({ getService, getPageObjects }: FtrProviderContext) { await rowActions[idxToClick].click(); }); - const hasDocHit = await testSubjects.exists('doc-hit'); - expect(hasDocHit).to.be(true); - const runtimeFieldsRow = await testSubjects.exists('tableDocViewRow-discover runtimefield'); - expect(runtimeFieldsRow).to.be(true); + await retry.waitFor('doc viewer is displayed with runtime field', async () => { + const hasDocHit = await testSubjects.exists('doc-hit'); + if (!hasDocHit) { + // Maybe loading has not completed + throw new Error('test subject doc-hit is not yet displayed'); + } + const runtimeFieldsRow = await testSubjects.exists('tableDocViewRow-discover runtimefield'); + + return hasDocHit && runtimeFieldsRow; + }); }); }); } From 8f5dad98a181eed7c4b1efe6c1cf41f906cfd042 Mon Sep 17 00:00:00 2001 From: Kerry Gallagher <471693+Kerry350@users.noreply.github.com> Date: Mon, 14 Jun 2021 19:28:08 +0100 Subject: [PATCH 12/77] [Logs / Metrics UI] Convert logs and metrics pages to the new Observability page template (#101239) * Convert Logs and Metrics pages to use the Observability page template --- x-pack/plugins/infra/kibana.json | 5 +- .../metric_anomaly/components/expression.tsx | 4 +- .../infra/public/apps/common_styles.ts | 13 +- x-pack/plugins/infra/public/apps/logs_app.tsx | 2 +- .../plugins/infra/public/apps/metrics_app.tsx | 6 +- .../public/assets/anomaly_chart_minified.svg | 1 - .../components/empty_states/no_indices.tsx | 28 +-- .../infra/public/components/error_page.tsx | 97 +++----- .../infra/public/components/loading_page.tsx | 44 ++-- .../logging/log_analysis_setup/index.ts | 2 - .../logging/log_analysis_setup/setup_page.tsx | 60 ----- .../logging/log_source_error_page.tsx | 14 +- .../components/navigation/app_navigation.tsx | 33 --- .../components/navigation/routed_tabs.tsx | 62 ----- .../infra/public/components/page_template.tsx | 22 ++ .../subscription_splash_content.tsx | 123 +++------- .../infra/public/components/toolbar_panel.ts | 27 -- x-pack/plugins/infra/public/index.scss | 16 -- .../pages/link_to/link_to_logs.test.tsx | 16 +- .../pages/logs/log_entry_categories/page.tsx | 5 +- .../log_entry_categories/page_content.tsx | 61 ++++- .../page_results_content.tsx | 77 ++++-- .../page_setup_content.tsx | 34 ++- .../top_categories/top_categories_section.tsx | 68 +---- .../public/pages/logs/log_entry_rate/page.tsx | 5 +- .../logs/log_entry_rate/page_content.tsx | 58 ++++- .../log_entry_rate/page_results_content.tsx | 160 ++++++------ .../log_entry_rate/page_setup_content.tsx | 34 ++- .../sections/anomalies/index.tsx | 19 -- .../infra/public/pages/logs/page_content.tsx | 25 +- .../infra/public/pages/logs/page_template.tsx | 24 ++ .../source_configuration_settings.tsx | 196 +++++++-------- .../infra/public/pages/logs/stream/page.tsx | 7 +- .../public/pages/logs/stream/page_content.tsx | 33 ++- .../pages/logs/stream/page_logs_content.tsx | 4 +- .../public/pages/logs/stream/page_toolbar.tsx | 7 +- .../infra/public/pages/metrics/index.tsx | 199 +++++++-------- .../components/bottom_drawer.tsx | 4 +- .../inventory_view/components/filter_bar.tsx | 24 +- .../ml/anomaly_detection/flyout_home.tsx | 8 +- .../waffle/waffle_time_controls.tsx | 13 +- .../pages/metrics/inventory_view/index.tsx | 157 +++++++----- .../components/node_details_page.tsx | 118 ++++----- .../metric_detail/components/side_nav.tsx | 25 +- .../pages/metrics/metric_detail/index.tsx | 80 +++--- .../metrics_explorer/components/toolbar.tsx | 5 +- .../pages/metrics/metrics_explorer/index.tsx | 75 +++--- .../public/pages/metrics/page_template.tsx | 24 ++ .../source_configuration_settings.tsx | 232 +++++++++--------- x-pack/plugins/infra/public/plugin.ts | 59 ++++- .../public/components/shared/index.tsx | 1 + .../components/shared/page_template/index.ts | 1 + .../shared/page_template/page_template.tsx | 1 + x-pack/plugins/observability/public/index.ts | 2 + .../translations/translations/ja-JP.json | 6 - .../translations/translations/zh-CN.json | 6 - .../page_objects/infra_home_page.ts | 24 +- 57 files changed, 1149 insertions(+), 1307 deletions(-) delete mode 100644 x-pack/plugins/infra/public/assets/anomaly_chart_minified.svg delete mode 100644 x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_page.tsx delete mode 100644 x-pack/plugins/infra/public/components/navigation/app_navigation.tsx delete mode 100644 x-pack/plugins/infra/public/components/navigation/routed_tabs.tsx create mode 100644 x-pack/plugins/infra/public/components/page_template.tsx delete mode 100644 x-pack/plugins/infra/public/components/toolbar_panel.ts create mode 100644 x-pack/plugins/infra/public/pages/logs/page_template.tsx create mode 100644 x-pack/plugins/infra/public/pages/metrics/page_template.tsx diff --git a/x-pack/plugins/infra/kibana.json b/x-pack/plugins/infra/kibana.json index c0d567ef83cedb..ec1b11c90f7a31 100644 --- a/x-pack/plugins/infra/kibana.json +++ b/x-pack/plugins/infra/kibana.json @@ -11,9 +11,10 @@ "dataEnhanced", "visTypeTimeseries", "alerting", - "triggersActionsUi" + "triggersActionsUi", + "observability" ], - "optionalPlugins": ["ml", "observability", "home", "embeddable"], + "optionalPlugins": ["ml", "home", "embeddable"], "server": true, "ui": true, "configPath": ["xpack", "infra"], diff --git a/x-pack/plugins/infra/public/alerting/metric_anomaly/components/expression.tsx b/x-pack/plugins/infra/public/alerting/metric_anomaly/components/expression.tsx index afbd6ffa8b5f7e..e44a747aa07e71 100644 --- a/x-pack/plugins/infra/public/alerting/metric_anomaly/components/expression.tsx +++ b/x-pack/plugins/infra/public/alerting/metric_anomaly/components/expression.tsx @@ -11,7 +11,7 @@ import { EuiFlexGroup, EuiSpacer, EuiText, EuiLoadingContent } from '@elastic/eu import { FormattedMessage } from '@kbn/i18n/react'; import { i18n } from '@kbn/i18n'; import { useInfraMLCapabilities } from '../../../containers/ml/infra_ml_capabilities'; -import { SubscriptionSplashContent } from '../../../components/subscription_splash_content'; +import { SubscriptionSplashPrompt } from '../../../components/subscription_splash_content'; import { AlertPreview } from '../../common'; import { METRIC_ANOMALY_ALERT_TYPE_ID, @@ -185,7 +185,7 @@ export const Expression: React.FC = (props) => { }, [metadata, derivedIndexPattern, defaultExpression, source, space]); // eslint-disable-line react-hooks/exhaustive-deps if (isLoadingMLCapabilities) return ; - if (!hasInfraMLCapabilities) return ; + if (!hasInfraMLCapabilities) return ; return ( // https://github.com/elastic/kibana/issues/89506 diff --git a/x-pack/plugins/infra/public/apps/common_styles.ts b/x-pack/plugins/infra/public/apps/common_styles.ts index be12c6cdc937fd..68c820d538ca9b 100644 --- a/x-pack/plugins/infra/public/apps/common_styles.ts +++ b/x-pack/plugins/infra/public/apps/common_styles.ts @@ -5,10 +5,15 @@ * 2.0. */ +import { APP_WRAPPER_CLASS } from '../../../../../src/core/public'; + export const CONTAINER_CLASSNAME = 'infra-container-element'; -export const prepareMountElement = (element: HTMLElement) => { - // Ensure the element we're handed from application mounting is assigned a class - // for our index.scss styles to apply to. - element.classList.add(CONTAINER_CLASSNAME); +export const prepareMountElement = (element: HTMLElement, testSubject?: string) => { + // Ensure all wrapping elements have the APP_WRAPPER_CLASS so that the KinanaPageTemplate works as expected + element.classList.add(APP_WRAPPER_CLASS); + + if (testSubject) { + element.setAttribute('data-test-subj', testSubject); + } }; diff --git a/x-pack/plugins/infra/public/apps/logs_app.tsx b/x-pack/plugins/infra/public/apps/logs_app.tsx index 61082efe436473..b512b5ce4a1764 100644 --- a/x-pack/plugins/infra/public/apps/logs_app.tsx +++ b/x-pack/plugins/infra/public/apps/logs_app.tsx @@ -27,7 +27,7 @@ export const renderApp = ( ) => { const storage = new Storage(window.localStorage); - prepareMountElement(element); + prepareMountElement(element, 'infraLogsPage'); ReactDOM.render( { const storage = new Storage(window.localStorage); - prepareMountElement(element); + prepareMountElement(element, 'infraMetricsPage'); ReactDOM.render( )} - {uiCapabilities?.infrastructure?.show && ( - - )} {uiCapabilities?.infrastructure?.show && ( )} diff --git a/x-pack/plugins/infra/public/assets/anomaly_chart_minified.svg b/x-pack/plugins/infra/public/assets/anomaly_chart_minified.svg deleted file mode 100644 index dd1b39248bba25..00000000000000 --- a/x-pack/plugins/infra/public/assets/anomaly_chart_minified.svg +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file diff --git a/x-pack/plugins/infra/public/components/empty_states/no_indices.tsx b/x-pack/plugins/infra/public/components/empty_states/no_indices.tsx index 264428e44a44ad..c61a567ac73b15 100644 --- a/x-pack/plugins/infra/public/components/empty_states/no_indices.tsx +++ b/x-pack/plugins/infra/public/components/empty_states/no_indices.tsx @@ -7,8 +7,7 @@ import { EuiEmptyPrompt } from '@elastic/eui'; import React from 'react'; - -import { euiStyled } from '../../../../../../src/plugins/kibana_react/common'; +import { PageTemplate } from '../page_template'; interface NoIndicesProps { message: string; @@ -17,15 +16,16 @@ interface NoIndicesProps { 'data-test-subj'?: string; } -export const NoIndices: React.FC = ({ actions, message, title, ...rest }) => ( - {title}} - body={

{message}

} - actions={actions} - {...rest} - /> -); - -const CenteredEmptyPrompt = euiStyled(EuiEmptyPrompt)` - align-self: center; -`; +// Represents a fully constructed page, including page template. +export const NoIndices: React.FC = ({ actions, message, title, ...rest }) => { + return ( + + {title}} + body={

{message}

} + actions={actions} + {...rest} + /> + + ); +}; diff --git a/x-pack/plugins/infra/public/components/error_page.tsx b/x-pack/plugins/infra/public/components/error_page.tsx index 184901b4fdd9b0..da6716ddc7f723 100644 --- a/x-pack/plugins/infra/public/components/error_page.tsx +++ b/x-pack/plugins/infra/public/components/error_page.tsx @@ -5,20 +5,10 @@ * 2.0. */ -import { - EuiButton, - EuiCallOut, - EuiFlexGroup, - EuiFlexItem, - EuiPageBody, - EuiPageContent, - EuiPageContentBody, - EuiSpacer, -} from '@elastic/eui'; +import { EuiButton, EuiCallOut, EuiFlexGroup, EuiFlexItem, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import React from 'react'; -import { euiStyled } from '../../../../../src/plugins/kibana_react/common'; -import { FlexPage } from './page'; +import { PageTemplate } from './page_template'; interface Props { detailedMessage?: React.ReactNode; @@ -26,51 +16,40 @@ interface Props { shortMessage: React.ReactNode; } -export const ErrorPage: React.FC = ({ detailedMessage, retry, shortMessage }) => ( - - - = ({ detailedMessage, retry, shortMessage }) => { + return ( + + + } > - - - } - > - - {shortMessage} - {retry ? ( - - - - - - ) : null} - - {detailedMessage ? ( - <> - -
{detailedMessage}
- - ) : null} - - - - - -); - -const MinimumPageContent = euiStyled(EuiPageContent)` - min-width: 50vh; -`; + + {shortMessage} + {retry ? ( + + + + + + ) : null} + + {detailedMessage ? ( + <> + +
{detailedMessage}
+ + ) : null} + + + ); +}; diff --git a/x-pack/plugins/infra/public/components/loading_page.tsx b/x-pack/plugins/infra/public/components/loading_page.tsx index 755511374b75fa..2b2859707a20d7 100644 --- a/x-pack/plugins/infra/public/components/loading_page.tsx +++ b/x-pack/plugins/infra/public/components/loading_page.tsx @@ -5,34 +5,38 @@ * 2.0. */ -import { - EuiFlexGroup, - EuiFlexItem, - EuiLoadingSpinner, - EuiPageBody, - EuiPageContent, -} from '@elastic/eui'; +import { EuiEmptyPrompt, EuiLoadingSpinner, EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; import React, { ReactNode } from 'react'; - -import { FlexPage } from './page'; +import { PageTemplate } from './page_template'; interface LoadingPageProps { message?: ReactNode; 'data-test-subj'?: string; } +// Represents a fully constructed page, including page template. export const LoadingPage = ({ message, 'data-test-subj': dataTestSubj = 'loadingPage', -}: LoadingPageProps) => ( - - - - - - {message} +}: LoadingPageProps) => { + return ( + + + + ); +}; + +export const LoadingPrompt = ({ message }: LoadingPageProps) => { + return ( + + + + + {message} - - - -); + } + /> + ); +}; diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/index.ts b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/index.ts index db5a996c604fcd..9ca08c69cf6006 100644 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/index.ts +++ b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/index.ts @@ -5,8 +5,6 @@ * 2.0. */ -export * from './setup_page'; - export * from './initial_configuration_step'; export * from './process_step'; diff --git a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_page.tsx b/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_page.tsx deleted file mode 100644 index a998d0c304a5ee..00000000000000 --- a/x-pack/plugins/infra/public/components/logging/log_analysis_setup/setup_page.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { - CommonProps, - EuiPage, - EuiPageBody, - EuiPageContent, - EuiPageContentBody, - EuiPageContentHeader, - EuiPageContentHeaderSection, - EuiTitle, -} from '@elastic/eui'; -import React from 'react'; - -import { euiStyled } from '../../../../../../../src/plugins/kibana_react/common'; - -export const LogAnalysisSetupPage: React.FunctionComponent = ({ - children, - ...rest -}) => { - return ( - - - - {children} - - - - ); -}; - -export const LogAnalysisSetupPageHeader: React.FunctionComponent = ({ children }) => ( - - - -

{children}

- - - -); - -export const LogAnalysisSetupPageContent = EuiPageContentBody; - -// !important due to https://github.com/elastic/eui/issues/2232 -const LogEntryRateSetupPageContent = euiStyled(EuiPageContent)` - max-width: 768px !important; -`; - -const LogEntryRateSetupPage = euiStyled(EuiPage)` - height: 100%; -`; diff --git a/x-pack/plugins/infra/public/components/logging/log_source_error_page.tsx b/x-pack/plugins/infra/public/components/logging/log_source_error_page.tsx index 8ea35fd8f259f7..6c757f7383a060 100644 --- a/x-pack/plugins/infra/public/components/logging/log_source_error_page.tsx +++ b/x-pack/plugins/infra/public/components/logging/log_source_error_page.tsx @@ -5,14 +5,7 @@ * 2.0. */ -import { - EuiButton, - EuiButtonEmpty, - EuiCallOut, - EuiEmptyPrompt, - EuiPageTemplate, - EuiSpacer, -} from '@elastic/eui'; +import { EuiButton, EuiButtonEmpty, EuiCallOut, EuiEmptyPrompt, EuiSpacer } from '@elastic/eui'; import { FormattedMessage } from '@kbn/i18n/react'; import React from 'react'; import { SavedObjectNotFound } from '../../../../../../src/plugins/kibana_utils/common'; @@ -22,6 +15,7 @@ import { ResolveLogSourceConfigurationError, } from '../../../common/log_sources'; import { useLinkProps } from '../../hooks/use_link_props'; +import { LogsPageTemplate } from '../../pages/logs/page_template'; export const LogSourceErrorPage: React.FC<{ errors: Error[]; @@ -30,7 +24,7 @@ export const LogSourceErrorPage: React.FC<{ const settingsLinkProps = useLinkProps({ app: 'logs', pathname: '/settings' }); return ( - + , ]} /> - + ); }; diff --git a/x-pack/plugins/infra/public/components/navigation/app_navigation.tsx b/x-pack/plugins/infra/public/components/navigation/app_navigation.tsx deleted file mode 100644 index 966b91537d3bd8..00000000000000 --- a/x-pack/plugins/infra/public/components/navigation/app_navigation.tsx +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiFlexGroup, EuiFlexItem } from '@elastic/eui'; -import React from 'react'; -import { euiStyled } from '../../../../../../src/plugins/kibana_react/common'; - -interface AppNavigationProps { - 'aria-label': string; - children: React.ReactNode; -} - -export const AppNavigation = ({ 'aria-label': label, children }: AppNavigationProps) => ( - -); - -const Nav = euiStyled.nav` - background: ${(props) => props.theme.eui.euiColorEmptyShade}; - border-bottom: ${(props) => props.theme.eui.euiBorderThin}; - padding: ${(props) => `${props.theme.eui.euiSizeS} ${props.theme.eui.euiSizeL}`}; - .euiTabs { - padding-left: 3px; - margin-left: -3px; - }; -`; diff --git a/x-pack/plugins/infra/public/components/navigation/routed_tabs.tsx b/x-pack/plugins/infra/public/components/navigation/routed_tabs.tsx deleted file mode 100644 index 2a5ffcd826e7c3..00000000000000 --- a/x-pack/plugins/infra/public/components/navigation/routed_tabs.tsx +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -import { EuiLink, EuiTab, EuiTabs } from '@elastic/eui'; -import React from 'react'; -import { Route } from 'react-router-dom'; - -import { euiStyled } from '../../../../../../src/plugins/kibana_react/common'; -import { useLinkProps } from '../../hooks/use_link_props'; -import { LinkDescriptor } from '../../hooks/use_link_props'; - -interface TabConfig { - title: string | React.ReactNode; -} - -type TabConfiguration = TabConfig & LinkDescriptor; - -interface RoutedTabsProps { - tabs: TabConfiguration[]; -} - -const noop = () => {}; - -export const RoutedTabs = ({ tabs }: RoutedTabsProps) => { - return ( - - {tabs.map((tab) => { - return ; - })} - - ); -}; - -const Tab = ({ title, pathname, app }: TabConfiguration) => { - const linkProps = useLinkProps({ app, pathname }); - return ( - { - return ( - - - - {title} - - - - ); - }} - /> - ); -}; - -const TabContainer = euiStyled.div` - .euiLink { - color: inherit !important; - } -`; diff --git a/x-pack/plugins/infra/public/components/page_template.tsx b/x-pack/plugins/infra/public/components/page_template.tsx new file mode 100644 index 00000000000000..1a10a6cd831b91 --- /dev/null +++ b/x-pack/plugins/infra/public/components/page_template.tsx @@ -0,0 +1,22 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import React from 'react'; +import { useKibanaContextForPlugin } from '../hooks/use_kibana'; +import type { LazyObservabilityPageTemplateProps } from '../../../observability/public'; + +export const PageTemplate: React.FC = (pageTemplateProps) => { + const { + services: { + observability: { + navigation: { PageTemplate: Template }, + }, + }, + } = useKibanaContextForPlugin(); + + return