diff --git a/x-pack/legacy/plugins/apm/server/lib/apm_telemetry/telemetry_metrics_tasks/index.ts b/x-pack/legacy/plugins/apm/server/lib/apm_telemetry/telemetry_metrics_tasks/index.ts new file mode 100644 index 000000000000000..e9cf66d4a138c6d --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/apm_telemetry/telemetry_metrics_tasks/index.ts @@ -0,0 +1,409 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { Transaction } from '../../../../typings/es_schemas/ui/Transaction'; +import { + PROCESSOR_EVENT, + SERVICE_NAME, + SERVICE_AGENT_NAME, + ERROR_GROUP_ID, + TRANSACTION_NAME, + PARENT_ID, + TRACE_ID +} from '../../../../common/elasticsearch_fieldnames'; +import { + ESSearchRequest, + ESSearchResponse +} from '../../../../typings/elasticsearch'; +import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; +import { APMDataTelemetry, TimeframeMap } from '../types'; +import { Span } from '../../../../typings/es_schemas/ui/Span'; +import { APMError } from '../../../../typings/es_schemas/ui/APMError'; + +type TelemetryTaskExecutor = ({ + search +}: { + indices: ApmIndicesConfig; + search( + params: TSearchRequest + ): Promise>; +}) => Promise; + +export interface TelemetryTask { + name: string; + executor: TelemetryTaskExecutor; +} + +export const telemetryMetricsTasks: TelemetryTask[] = [ + { + name: 'processor_events', + executor: async ({ indices, search }) => { + const response = await search({ + index: [ + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.transactionIndices'], + indices['apm_oss.onboardingIndices'], + indices['apm_oss.sourcemapIndices'] + ], + body: { + size: 0, + aggs: { + processor_events: { + filter: { + exists: { + field: PROCESSOR_EVENT + } + }, + aggs: { + by_event_type: { + terms: { + field: PROCESSOR_EVENT, + size: 6 + }, + aggs: { + ranges: { + date_range: { + field: '@timestamp', + ranges: [ + { from: 'now-1d', key: '1d' }, + { from: 'now-1M', key: '1M' }, + { from: 'now-6M', key: '6M' } + ], + keyed: true + } + } + } + } + } + } + } + } + }); + + if (!response.aggregations) { + return {}; + } + + return { + counts: response.aggregations.processor_events.by_event_type.buckets.reduce( + (acc, bucket) => { + return { + ...acc, + [bucket.key as string]: { + '1d': bucket.ranges.buckets['1d'].doc_count, + '1M': bucket.ranges.buckets['1M'].doc_count, + '6M': bucket.ranges.buckets['6M'].doc_count, + all: bucket.doc_count + } + }; + }, + {} as Record< + | 'transaction' + | 'span' + | 'metrics' + | 'error' + | 'onboarding' + | 'sourcemap', + TimeframeMap + > + ) + }; + } + }, + { + name: 'agent_configuration', + executor: async ({ indices, search }) => { + const agentConfigurationCount = ( + await search({ + index: indices.apmAgentConfigurationIndex, + body: { + size: 0, + track_total_hits: true + } + }) + ).hits.total.value; + + return { + counts: { + agent_configuration: { + all: agentConfigurationCount + } + } + }; + } + }, + { + name: 'services', + executor: async ({ indices, search }) => { + const response = await search({ + index: [ + indices['apm_oss.errorIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.transactionIndices'] + ], + body: { + size: 0, + aggs: { + agents: { + terms: { + field: SERVICE_AGENT_NAME + }, + aggs: { + services: { + cardinality: { + field: SERVICE_NAME + } + } + } + } + } + } + }); + + if (!response.aggregations) { + return {}; + } + + const servicesByAgentName = response.aggregations.agents.buckets.reduce( + (prev, bucket) => { + const agentName = bucket.key as keyof typeof prev; + if (agentName in prev) { + prev[agentName] = bucket.services.value; + } + return prev; + }, + { + 'js-base': 0, + 'rum-js': 0, + dotnet: 0, + go: 0, + java: 0, + nodejs: 0, + python: 0, + ruby: 0 + } + ); + + return { + has_any_services: response.aggregations.agents.buckets.length > 0, + services_per_agent: { + 'js-base': servicesByAgentName['js-base'], + 'rum-js': servicesByAgentName['rum-js'], + dotnet: servicesByAgentName.dotnet, + go: servicesByAgentName.go, + java: servicesByAgentName.java, + nodejs: servicesByAgentName.nodejs, + python: servicesByAgentName.python, + ruby: servicesByAgentName.ruby + } + }; + } + }, + { + name: 'versions', + executor: async ({ search, indices }) => { + const response = await search({ + index: [ + indices['apm_oss.transactionIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.errorIndices'] + ], + body: { + query: { + exists: { + field: 'observer.version' + } + }, + size: 1, + sort: { + '@timestamp': 'desc' + } + } + }); + + const hit = response.hits.hits[0]?._source as Pick< + Transaction | Span | APMError, + 'observer' + >; + + if (!hit || !hit.observer?.version) { + return {}; + } + + const [major, minor, patch] = hit.observer.version + .split('.') + .map(part => Number(part)); + + return { + versions: { + apm_server: { + major, + minor, + patch + } + } + }; + } + }, + { + name: 'groupings', + executor: async ({ search, indices }) => { + const errorGroupsCount = ( + await search({ + index: indices['apm_oss.errorIndices'], + body: { + query: { + bool: { + filter: [{ term: { [PROCESSOR_EVENT]: 'error' } }] + } + }, + aggs: { + top_service: { + terms: { + field: SERVICE_NAME, + order: { + error_groups: 'desc' + }, + size: 1 + }, + aggs: { + error_groups: { + cardinality: { + field: ERROR_GROUP_ID + } + } + } + } + } + } + }) + ).aggregations?.top_service.buckets[0]?.error_groups.value; + + const transactionGroupsCount = ( + await search({ + index: indices['apm_oss.transactionIndices'], + body: { + query: { + bool: { + filter: [{ term: { [PROCESSOR_EVENT]: 'transaction' } }] + } + }, + aggs: { + top_service: { + terms: { + field: SERVICE_NAME, + order: { + transaction_groups: 'desc' + }, + size: 1 + }, + aggs: { + transaction_groups: { + cardinality: { + field: TRANSACTION_NAME + } + } + } + } + } + } + }) + ).aggregations?.top_service.buckets[0]?.transaction_groups.value; + + const tracesCount = ( + await search({ + index: indices['apm_oss.transactionIndices'], + body: { + query: { + bool: { + filter: [{ term: { [PROCESSOR_EVENT]: 'transaction' } }], + must_not: { + exists: { field: PARENT_ID } + } + } + }, + aggs: { + top_service: { + terms: { + field: SERVICE_NAME, + order: { + traces: 'desc' + }, + size: 1 + }, + aggs: { + traces: { + cardinality: { + field: TRACE_ID + } + } + } + } + } + } + }) + ).aggregations?.top_service.buckets[0]?.traces.value; + + const servicesCount = ( + await search({ + index: [ + indices['apm_oss.transactionIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'] + ], + body: { + aggs: { + service_name: { + cardinality: { + field: SERVICE_NAME + } + } + } + } + }) + ).aggregations?.service_name.value; + + return { + counts: { + transaction_group: { + all: transactionGroupsCount || 0 + }, + error_group: { + all: errorGroupsCount || 0 + }, + trace: { + all: tracesCount || 0 + }, + service: { + all: servicesCount || 0 + } + } + }; + } + }, + { + name: 'integrations', + executor: async ({ search }) => { + const apmJobs = ['high_mean_response_time']; + + const response = await search({ + index: apmJobs.map(job => `.ml-anomalies-*-${job}`), + body: { + size: 1 + } + }); + + return { + integrations: { + ml: response.hits.total.value > 0 + } + }; + } + } +]; diff --git a/x-pack/legacy/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/legacy/plugins/apm/server/lib/apm_telemetry/types.ts new file mode 100644 index 000000000000000..c549e3b28834eca --- /dev/null +++ b/x-pack/legacy/plugins/apm/server/lib/apm_telemetry/types.ts @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { DeepPartial } from 'utility-types'; + +export interface TimeframeMap { + '1d': number; + '1M': number; + '6M': number; + all: number; +} + +export type TimeframeAll = Pick; + +export type APMDataTelemetry = DeepPartial<{ + has_any_services: boolean; + services_per_agent: { + go: number; + java: number; + 'js-base': number; + 'rum-js': number; + nodejs: number; + python: number; + dotnet: number; + ruby: number; + }; + versions: { + apm_server: { + minor: number; + major: number; + patch: number; + }; + }; + counts: { + transaction: TimeframeMap; + span: TimeframeMap; + error: TimeframeMap; + metrics: TimeframeMap; + sourcemap: TimeframeMap; + onboarding: TimeframeMap; + agent_configuration: TimeframeAll; + transaction_group: TimeframeAll; + error_group: TimeframeAll; + trace: TimeframeAll; + service: TimeframeAll; + }; + integrations: { + alerting: boolean; + ml: boolean; + }; +}>; + +export interface APMPerformanceMeasurement { + path: string; + response_time: { + ms: number; + }; + status_code: number; +} + +export interface APMPerformanceTelemetry { + total_requests: number; + total_response_time: { + ms: number; + }; + requests: APMPerformanceMeasurement[]; +} + +export type PerformanceMeasurement = APMPerformanceTelemetry['requests'][0]; + +export type APMTelemetry = APMDataTelemetry & APMPerformanceTelemetry; diff --git a/x-pack/legacy/plugins/apm/typings/es_schemas/raw/fields/Observer.ts b/x-pack/legacy/plugins/apm/typings/es_schemas/raw/fields/Observer.ts new file mode 100644 index 000000000000000..42843130ec47fb3 --- /dev/null +++ b/x-pack/legacy/plugins/apm/typings/es_schemas/raw/fields/Observer.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface Observer { + version: string; + version_major: number; +}