diff --git a/x-pack/plugins/apm/common/projections/services.ts b/x-pack/plugins/apm/common/projections/services.ts index 80a3471e9c30d4..809caeeaf60885 100644 --- a/x-pack/plugins/apm/common/projections/services.ts +++ b/x-pack/plugins/apm/common/projections/services.ts @@ -16,25 +16,37 @@ import { rangeFilter } from '../utils/range_filter'; export function getServicesProjection({ setup, + noEvents, }: { setup: Setup & SetupTimeRange & SetupUIFilters; + noEvents?: boolean; }) { const { start, end, uiFiltersES, indices } = setup; return { - index: [ - indices['apm_oss.metricsIndices'], - indices['apm_oss.errorIndices'], - indices['apm_oss.transactionIndices'], - ], + ...(noEvents + ? {} + : { + index: [ + indices['apm_oss.metricsIndices'], + indices['apm_oss.errorIndices'], + indices['apm_oss.transactionIndices'], + ], + }), body: { size: 0, query: { bool: { filter: [ - { - terms: { [PROCESSOR_EVENT]: ['transaction', 'error', 'metric'] }, - }, + ...(noEvents + ? [] + : [ + { + terms: { + [PROCESSOR_EVENT]: ['transaction', 'error', 'metric'], + }, + }, + ]), { range: rangeFilter(start, end) }, ...uiFiltersES, ], diff --git a/x-pack/plugins/apm/common/utils/array_union_to_callable.ts b/x-pack/plugins/apm/common/utils/array_union_to_callable.ts new file mode 100644 index 00000000000000..23ea86006b888a --- /dev/null +++ b/x-pack/plugins/apm/common/utils/array_union_to_callable.ts @@ -0,0 +1,14 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { ValuesType } from 'utility-types'; + +// work around a TypeScript limitation described in https://stackoverflow.com/posts/49511416 + +export const arrayUnionToCallable = ( + array: T +): Array> => { + return array; +}; diff --git a/x-pack/plugins/apm/common/utils/join_by_key/index.test.ts b/x-pack/plugins/apm/common/utils/join_by_key/index.test.ts new file mode 100644 index 00000000000000..458d21bfea58fb --- /dev/null +++ b/x-pack/plugins/apm/common/utils/join_by_key/index.test.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { joinByKey } from './'; + +describe('joinByKey', () => { + it('joins by a string key', () => { + const joined = joinByKey( + [ + { + serviceName: 'opbeans-node', + avg: 10, + }, + { + serviceName: 'opbeans-node', + count: 12, + }, + { + serviceName: 'opbeans-java', + avg: 11, + }, + { + serviceName: 'opbeans-java', + p95: 18, + }, + ], + 'serviceName' + ); + + expect(joined.length).toBe(2); + + expect(joined).toEqual([ + { + serviceName: 'opbeans-node', + avg: 10, + count: 12, + }, + { + serviceName: 'opbeans-java', + avg: 11, + p95: 18, + }, + ]); + }); + + it('joins by a record key', () => { + const joined = joinByKey( + [ + { + key: { + serviceName: 'opbeans-node', + transactionName: '/api/opbeans-node', + }, + avg: 10, + }, + { + key: { + serviceName: 'opbeans-node', + transactionName: '/api/opbeans-node', + }, + count: 12, + }, + { + key: { + serviceName: 'opbeans-java', + transactionName: '/api/opbeans-java', + }, + avg: 11, + }, + { + key: { + serviceName: 'opbeans-java', + transactionName: '/api/opbeans-java', + }, + p95: 18, + }, + ], + 'key' + ); + + expect(joined.length).toBe(2); + + expect(joined).toEqual([ + { + key: { + serviceName: 'opbeans-node', + transactionName: '/api/opbeans-node', + }, + avg: 10, + count: 12, + }, + { + key: { + serviceName: 'opbeans-java', + transactionName: '/api/opbeans-java', + }, + avg: 11, + p95: 18, + }, + ]); + }); +}); diff --git a/x-pack/plugins/apm/common/utils/join_by_key/index.ts b/x-pack/plugins/apm/common/utils/join_by_key/index.ts new file mode 100644 index 00000000000000..b49f5364005149 --- /dev/null +++ b/x-pack/plugins/apm/common/utils/join_by_key/index.ts @@ -0,0 +1,48 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { UnionToIntersection, ValuesType } from 'utility-types'; +import { isEqual } from 'lodash'; + +/** + * Joins a list of records by a given key. Key can be any type of value, from + * strings to plain objects, as long as it is present in all records. `isEqual` + * is used for comparing keys. + * + * UnionToIntersection is needed to get all keys of union types, see below for + * example. + * + const agentNames = [{ serviceName: '', agentName: '' }]; + const transactionRates = [{ serviceName: '', transactionsPerMinute: 1 }]; + const flattened = joinByKey( + [...agentNames, ...transactionRates], + 'serviceName' + ); +*/ + +type JoinedReturnType< + T extends Record, + U extends UnionToIntersection, + V extends keyof T & keyof U +> = Array & Record>; + +export function joinByKey< + T extends Record, + U extends UnionToIntersection, + V extends keyof T & keyof U +>(items: T[], key: V): JoinedReturnType { + return items.reduce>((prev, current) => { + let item = prev.find((prevItem) => isEqual(prevItem[key], current[key])); + + if (!item) { + item = { ...current } as ValuesType>; + prev.push(item); + } else { + Object.assign(item, current); + } + + return prev; + }, []); +} diff --git a/x-pack/plugins/apm/common/utils/left_join.ts b/x-pack/plugins/apm/common/utils/left_join.ts deleted file mode 100644 index f3c4e48df755b0..00000000000000 --- a/x-pack/plugins/apm/common/utils/left_join.ts +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ -import { Assign, Omit } from 'utility-types'; - -export function leftJoin< - TL extends object, - K extends keyof TL, - TR extends Pick ->(leftRecords: TL[], matchKey: K, rightRecords: TR[]) { - const rightLookup = new Map( - rightRecords.map((record) => [record[matchKey], record]) - ); - return leftRecords.map((record) => { - const matchProp = (record[matchKey] as unknown) as TR[K]; - const matchingRightRecord = rightLookup.get(matchProp); - return { ...record, ...matchingRightRecord }; - }) as Array>>>; -} diff --git a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap index 3f8d6b22cd000b..0fc1f89a3723b3 100644 --- a/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap +++ b/x-pack/plugins/apm/server/lib/services/__snapshots__/queries.test.ts.snap @@ -113,76 +113,244 @@ Object { `; exports[`services queries fetches the service items 1`] = ` -Object { - "body": Object { - "aggs": Object { - "services": Object { - "aggs": Object { - "agents": Object { - "terms": Object { - "field": "agent.name", - "size": 1, +Array [ + Object { + "body": Object { + "aggs": Object { + "services": Object { + "aggs": Object { + "average": Object { + "avg": Object { + "field": "transaction.duration.us", + }, }, }, - "avg": Object { - "avg": Object { - "field": "transaction.duration.us", + "terms": Object { + "field": "service.name", + "size": 500, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "format": "epoch_millis", + "gte": 1528113600000, + "lte": 1528977600000, + }, + }, + }, + Object { + "term": Object { + "my.custom.ui.filter": "foo-bar", + }, + }, + Object { + "term": Object { + "processor.event": "transaction", + }, + }, + ], + }, + }, + "size": 0, + }, + "index": "myIndex", + "size": 0, + }, + Object { + "body": Object { + "aggs": Object { + "services": Object { + "aggs": Object { + "agent_name": Object { + "top_hits": Object { + "_source": Array [ + "agent.name", + ], + "size": 1, + }, }, }, - "environments": Object { - "terms": Object { - "field": "service.environment", + "terms": Object { + "field": "service.name", + "size": 500, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "format": "epoch_millis", + "gte": 1528113600000, + "lte": 1528977600000, + }, + }, + }, + Object { + "term": Object { + "my.custom.ui.filter": "foo-bar", + }, + }, + Object { + "terms": Object { + "processor.event": Array [ + "metric", + "error", + "transaction", + ], + }, }, + ], + }, + }, + "size": 0, + }, + "index": Array [ + "myIndex", + "myIndex", + "myIndex", + ], + }, + Object { + "body": Object { + "aggs": Object { + "services": Object { + "terms": Object { + "field": "service.name", + "size": 500, }, - "events": Object { - "terms": Object { - "field": "processor.event", + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "format": "epoch_millis", + "gte": 1528113600000, + "lte": 1528977600000, + }, + }, }, + Object { + "term": Object { + "my.custom.ui.filter": "foo-bar", + }, + }, + Object { + "term": Object { + "processor.event": "transaction", + }, + }, + ], + }, + }, + "size": 0, + }, + "index": "myIndex", + }, + Object { + "body": Object { + "aggs": Object { + "services": Object { + "terms": Object { + "field": "service.name", + "size": 500, }, }, - "terms": Object { - "field": "service.name", - "size": 500, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "format": "epoch_millis", + "gte": 1528113600000, + "lte": 1528977600000, + }, + }, + }, + Object { + "term": Object { + "my.custom.ui.filter": "foo-bar", + }, + }, + Object { + "term": Object { + "processor.event": "error", + }, + }, + ], }, }, + "size": 0, }, - "query": Object { - "bool": Object { - "filter": Array [ - Object { - "terms": Object { - "processor.event": Array [ - "transaction", - "error", - "metric", - ], + "index": "myIndex", + }, + Object { + "body": Object { + "aggs": Object { + "services": Object { + "aggs": Object { + "environments": Object { + "terms": Object { + "field": "service.environment", + }, }, }, - Object { - "range": Object { - "@timestamp": Object { - "format": "epoch_millis", - "gte": 1528113600000, - "lte": 1528977600000, + "terms": Object { + "field": "service.name", + "size": 500, + }, + }, + }, + "query": Object { + "bool": Object { + "filter": Array [ + Object { + "range": Object { + "@timestamp": Object { + "format": "epoch_millis", + "gte": 1528113600000, + "lte": 1528977600000, + }, }, }, - }, - Object { - "term": Object { - "my.custom.ui.filter": "foo-bar", + Object { + "term": Object { + "my.custom.ui.filter": "foo-bar", + }, }, - }, - ], + Object { + "terms": Object { + "processor.event": Array [ + "transaction", + "error", + "metric", + ], + }, + }, + ], + }, }, + "size": 0, }, - "size": 0, + "index": Array [ + "myIndex", + "myIndex", + "myIndex", + ], }, - "index": Array [ - "myIndex", - "myIndex", - "myIndex", - ], -} +] `; exports[`services queries fetches the service transaction types 1`] = ` diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts index acf052affabdb8..14772e77fe1c2c 100644 --- a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items.ts @@ -3,14 +3,7 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ - -import { mergeProjection } from '../../../../common/projections/util/merge_projection'; -import { - PROCESSOR_EVENT, - AGENT_NAME, - SERVICE_ENVIRONMENT, - TRANSACTION_DURATION, -} from '../../../../common/elasticsearch_fieldnames'; +import { joinByKey } from '../../../../common/utils/join_by_key'; import { PromiseReturnType } from '../../../../typings/common'; import { Setup, @@ -18,75 +11,45 @@ import { SetupUIFilters, } from '../../helpers/setup_request'; import { getServicesProjection } from '../../../../common/projections/services'; +import { + getTransactionDurationAverages, + getAgentNames, + getTransactionRates, + getErrorRates, + getEnvironments, +} from './get_services_items_stats'; export type ServiceListAPIResponse = PromiseReturnType; -export async function getServicesItems( - setup: Setup & SetupTimeRange & SetupUIFilters -) { - const { start, end, client } = setup; - - const projection = getServicesProjection({ setup }); - - const params = mergeProjection(projection, { - body: { - size: 0, - aggs: { - services: { - terms: { - ...projection.body.aggs.services.terms, - size: 500, - }, - aggs: { - avg: { - avg: { field: TRANSACTION_DURATION }, - }, - agents: { - terms: { field: AGENT_NAME, size: 1 }, - }, - events: { - terms: { field: PROCESSOR_EVENT }, - }, - environments: { - terms: { field: SERVICE_ENVIRONMENT }, - }, - }, - }, - }, - }, - }); - - const resp = await client.search(params); - const aggs = resp.aggregations; - - const serviceBuckets = aggs?.services.buckets || []; - - const items = serviceBuckets.map((bucket) => { - const eventTypes = bucket.events.buckets; - - const transactions = eventTypes.find((e) => e.key === 'transaction'); - const totalTransactions = transactions?.doc_count || 0; - - const errors = eventTypes.find((e) => e.key === 'error'); - const totalErrors = errors?.doc_count || 0; - - const deltaAsMinutes = (end - start) / 1000 / 60; - const transactionsPerMinute = totalTransactions / deltaAsMinutes; - const errorsPerMinute = totalErrors / deltaAsMinutes; - - const environmentsBuckets = bucket.environments.buckets; - const environments = environmentsBuckets.map( - (environmentBucket) => environmentBucket.key as string - ); - - return { - serviceName: bucket.key as string, - agentName: bucket.agents.buckets[0]?.key as string | undefined, - transactionsPerMinute, - errorsPerMinute, - avgResponseTime: bucket.avg.value, - environments, - }; - }); - - return items; +export type ServicesItemsSetup = Setup & SetupTimeRange & SetupUIFilters; +export type ServicesItemsProjection = ReturnType; + +export async function getServicesItems(setup: ServicesItemsSetup) { + const params = { + projection: getServicesProjection({ setup, noEvents: true }), + setup, + }; + + const [ + transactionDurationAverages, + agentNames, + transactionRates, + errorRates, + environments, + ] = await Promise.all([ + getTransactionDurationAverages(params), + getAgentNames(params), + getTransactionRates(params), + getErrorRates(params), + getEnvironments(params), + ]); + + const allMetrics = [ + ...transactionDurationAverages, + ...agentNames, + ...transactionRates, + ...errorRates, + ...environments, + ]; + + return joinByKey(allMetrics, 'serviceName'); } diff --git a/x-pack/plugins/apm/server/lib/services/get_services/get_services_items_stats.ts b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items_stats.ts new file mode 100644 index 00000000000000..c28bcad841ffd8 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/services/get_services/get_services_items_stats.ts @@ -0,0 +1,309 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { arrayUnionToCallable } from '../../../../common/utils/array_union_to_callable'; +import { + PROCESSOR_EVENT, + TRANSACTION_DURATION, + AGENT_NAME, + SERVICE_ENVIRONMENT, +} from '../../../../common/elasticsearch_fieldnames'; +import { mergeProjection } from '../../../../common/projections/util/merge_projection'; +import { ProcessorEvent } from '../../../../common/processor_event'; +import { + ServicesItemsSetup, + ServicesItemsProjection, +} from './get_services_items'; + +const MAX_NUMBER_OF_SERVICES = 500; + +const getDeltaAsMinutes = (setup: ServicesItemsSetup) => + (setup.end - setup.start) / 1000 / 60; + +interface AggregationParams { + setup: ServicesItemsSetup; + projection: ServicesItemsProjection; +} + +export const getTransactionDurationAverages = async ({ + setup, + projection, +}: AggregationParams) => { + const { client, indices } = setup; + + const response = await client.search( + mergeProjection(projection, { + size: 0, + index: indices['apm_oss.transactionIndices'], + body: { + query: { + bool: { + filter: projection.body.query.bool.filter.concat({ + term: { + [PROCESSOR_EVENT]: ProcessorEvent.transaction, + }, + }), + }, + }, + aggs: { + services: { + terms: { + ...projection.body.aggs.services.terms, + size: MAX_NUMBER_OF_SERVICES, + }, + aggs: { + average: { + avg: { + field: TRANSACTION_DURATION, + }, + }, + }, + }, + }, + }, + }) + ); + + const { aggregations } = response; + + if (!aggregations) { + return []; + } + + return aggregations.services.buckets.map((bucket) => ({ + serviceName: bucket.key as string, + avgResponseTime: bucket.average.value, + })); +}; + +export const getAgentNames = async ({ + setup, + projection, +}: AggregationParams) => { + const { client, indices } = setup; + const response = await client.search( + mergeProjection(projection, { + index: [ + indices['apm_oss.metricsIndices'], + indices['apm_oss.errorIndices'], + indices['apm_oss.transactionIndices'], + ], + body: { + size: 0, + query: { + bool: { + filter: [ + ...projection.body.query.bool.filter, + { + terms: { + [PROCESSOR_EVENT]: [ + ProcessorEvent.metric, + ProcessorEvent.error, + ProcessorEvent.transaction, + ], + }, + }, + ], + }, + }, + aggs: { + services: { + terms: { + ...projection.body.aggs.services.terms, + size: MAX_NUMBER_OF_SERVICES, + }, + aggs: { + agent_name: { + top_hits: { + _source: [AGENT_NAME], + size: 1, + }, + }, + }, + }, + }, + }, + }) + ); + + const { aggregations } = response; + + if (!aggregations) { + return []; + } + + return aggregations.services.buckets.map((bucket) => ({ + serviceName: bucket.key as string, + agentName: (bucket.agent_name.hits.hits[0]?._source as { + agent: { + name: string; + }; + }).agent.name, + })); +}; + +export const getTransactionRates = async ({ + setup, + projection, +}: AggregationParams) => { + const { client, indices } = setup; + const response = await client.search( + mergeProjection(projection, { + index: indices['apm_oss.transactionIndices'], + body: { + size: 0, + query: { + bool: { + filter: [ + ...projection.body.query.bool.filter, + { + term: { + [PROCESSOR_EVENT]: ProcessorEvent.transaction, + }, + }, + ], + }, + }, + aggs: { + services: { + terms: { + ...projection.body.aggs.services.terms, + size: MAX_NUMBER_OF_SERVICES, + }, + }, + }, + }, + }) + ); + + const { aggregations } = response; + + if (!aggregations) { + return []; + } + + const deltaAsMinutes = getDeltaAsMinutes(setup); + + return arrayUnionToCallable(aggregations.services.buckets).map((bucket) => { + const transactionsPerMinute = bucket.doc_count / deltaAsMinutes; + return { + serviceName: bucket.key as string, + transactionsPerMinute, + }; + }); +}; + +export const getErrorRates = async ({ + setup, + projection, +}: AggregationParams) => { + const { client, indices } = setup; + const response = await client.search( + mergeProjection(projection, { + index: indices['apm_oss.errorIndices'], + body: { + size: 0, + query: { + bool: { + filter: [ + ...projection.body.query.bool.filter, + { + term: { + [PROCESSOR_EVENT]: ProcessorEvent.error, + }, + }, + ], + }, + }, + aggs: { + services: { + terms: { + ...projection.body.aggs.services.terms, + size: MAX_NUMBER_OF_SERVICES, + }, + }, + }, + }, + }) + ); + + const { aggregations } = response; + + if (!aggregations) { + return []; + } + + const deltaAsMinutes = getDeltaAsMinutes(setup); + + return aggregations.services.buckets.map((bucket) => { + const errorsPerMinute = bucket.doc_count / deltaAsMinutes; + return { + serviceName: bucket.key as string, + errorsPerMinute, + }; + }); +}; + +export const getEnvironments = async ({ + setup, + projection, +}: AggregationParams) => { + const { client, indices } = setup; + const response = await client.search( + mergeProjection(projection, { + index: [ + indices['apm_oss.metricsIndices'], + indices['apm_oss.errorIndices'], + indices['apm_oss.transactionIndices'], + ], + body: { + size: 0, + query: { + bool: { + filter: [ + ...projection.body.query.bool.filter, + { + terms: { + [PROCESSOR_EVENT]: [ + ProcessorEvent.transaction, + ProcessorEvent.error, + ProcessorEvent.metric, + ], + }, + }, + ], + }, + }, + aggs: { + services: { + terms: { + ...projection.body.aggs.services.terms, + size: MAX_NUMBER_OF_SERVICES, + }, + aggs: { + environments: { + terms: { + field: SERVICE_ENVIRONMENT, + }, + }, + }, + }, + }, + }, + }) + ); + + const { aggregations } = response; + + if (!aggregations) { + return []; + } + + return aggregations.services.buckets.map((bucket) => ({ + serviceName: bucket.key as string, + environments: bucket.environments.buckets.map((env) => env.key as string), + })); +}; diff --git a/x-pack/plugins/apm/server/lib/services/queries.test.ts b/x-pack/plugins/apm/server/lib/services/queries.test.ts index d90cd8bf139080..b2fe7efeaf9599 100644 --- a/x-pack/plugins/apm/server/lib/services/queries.test.ts +++ b/x-pack/plugins/apm/server/lib/services/queries.test.ts @@ -40,7 +40,9 @@ describe('services queries', () => { it('fetches the service items', async () => { mock = await inspectSearchParams((setup) => getServicesItems(setup)); - expect(mock.params).toMatchSnapshot(); + const allParams = mock.spy.mock.calls.map((call) => call[0]); + + expect(allParams).toMatchSnapshot(); }); it('fetches the legacy data status', async () => {