diff --git a/x-pack/legacy/plugins/apm/index.ts b/x-pack/legacy/plugins/apm/index.ts index 0107997f233fe07..dbca321d813e7d2 100644 --- a/x-pack/legacy/plugins/apm/index.ts +++ b/x-pack/legacy/plugins/apm/index.ts @@ -14,7 +14,13 @@ import mappings from './mappings.json'; export const apm: LegacyPluginInitializer = kibana => { return new kibana.Plugin({ - require: ['kibana', 'elasticsearch', 'xpack_main', 'apm_oss'], + require: [ + 'kibana', + 'elasticsearch', + 'xpack_main', + 'apm_oss', + 'task_manager' + ], id: 'apm', configPrefix: 'xpack.apm', publicDir: resolve(__dirname, 'public'), @@ -107,10 +113,12 @@ export const apm: LegacyPluginInitializer = kibana => { } } }); - const apmPlugin = server.newPlatform.setup.plugins .apm as APMPluginContract; - apmPlugin.registerLegacyAPI({ server }); + + apmPlugin.registerLegacyAPI({ + server + }); } }); }; diff --git a/x-pack/legacy/plugins/apm/mappings.json b/x-pack/legacy/plugins/apm/mappings.json index 61bc90da2875670..03987003398c02f 100644 --- a/x-pack/legacy/plugins/apm/mappings.json +++ b/x-pack/legacy/plugins/apm/mappings.json @@ -1,46 +1,7 @@ { - "apm-services-telemetry": { - "properties": { - "has_any_services": { - "type": "boolean" - }, - "services_per_agent": { - "properties": { - "python": { - "type": "long", - "null_value": 0 - }, - "java": { - "type": "long", - "null_value": 0 - }, - "nodejs": { - "type": "long", - "null_value": 0 - }, - "js-base": { - "type": "long", - "null_value": 0 - }, - "rum-js": { - "type": "long", - "null_value": 0 - }, - "dotnet": { - "type": "long", - "null_value": 0 - }, - "ruby": { - "type": "long", - "null_value": 0 - }, - "go": { - "type": "long", - "null_value": 0 - } - } - } - } + "apm-telemetry": { + "properties": {}, + "dynamic": true }, "apm-indices": { "properties": { diff --git a/x-pack/legacy/plugins/apm/scripts/.gitignore b/x-pack/legacy/plugins/apm/scripts/.gitignore new file mode 100644 index 000000000000000..8ee01d321b721ad --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/.gitignore @@ -0,0 +1 @@ +yarn.lock diff --git a/x-pack/legacy/plugins/apm/scripts/package.json b/x-pack/legacy/plugins/apm/scripts/package.json new file mode 100644 index 000000000000000..9121449c5361992 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/package.json @@ -0,0 +1,10 @@ +{ + "name": "apm-scripts", + "version": "1.0.0", + "main": "index.js", + "license": "MIT", + "dependencies": { + "@octokit/rest": "^16.35.0", + "console-stamp": "^0.2.9" + } +} diff --git a/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js b/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js index 825c1a526fcc529..61ba2fdc7f7e307 100644 --- a/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js +++ b/x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js @@ -16,6 +16,7 @@ ******************************/ // compile typescript on the fly +// eslint-disable-next-line import/no-extraneous-dependencies require('@babel/register')({ extensions: ['.ts'], plugins: ['@babel/plugin-proposal-optional-chaining'], diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js new file mode 100644 index 000000000000000..a99651c62dd7ad9 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js @@ -0,0 +1,21 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +// compile typescript on the fly +// eslint-disable-next-line import/no-extraneous-dependencies +require('@babel/register')({ + extensions: ['.ts'], + plugins: [ + '@babel/plugin-proposal-optional-chaining', + '@babel/plugin-proposal-nullish-coalescing-operator' + ], + presets: [ + '@babel/typescript', + ['@babel/preset-env', { targets: { node: 'current' } }] + ] +}); + +require('./upload-telemetry-data/index.ts'); diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-mapping.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-mapping.ts new file mode 100644 index 000000000000000..45c44ee77655df7 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-mapping.ts @@ -0,0 +1,24 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import Ocktokit from '@octokit/rest'; + +export async function downloadTelemetryMapping(ocktokit: Ocktokit) { + const file = await ocktokit.repos.getContents({ + owner: 'elastic', + repo: 'telemetry', + path: 'config/templates/xpack-phone-home.json', + mediaType: { + format: 'application/vnd.github.VERSION.raw' + } + }); + + if (Array.isArray(file.data)) { + throw new Error('Expected single response, got array'); + } + + return JSON.parse(Buffer.from(file.data.content!, 'base64').toString()); +} diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts new file mode 100644 index 000000000000000..a02090930b1f1d9 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts @@ -0,0 +1,120 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { DeepPartial } from 'utility-types'; +import { + merge, + omit, + defaultsDeep, + range, + mapValues, + isPlainObject, + flatten +} from 'lodash'; +import uuid from 'uuid'; +import { + CollectTelemetryParams, + collectDataTelemetry +} from '../../../../../plugins/apm/server/lib/apm_telemetry/collect_data_telemetry'; + +interface GenerateOptions { + days: number; + instances: number; + variation: { + min: number; + max: number; + }; +} + +const randomize = ( + value: unknown, + instanceVariation: number, + dailyGrowth: number +) => { + if (typeof value === 'boolean') { + return Math.random() > 0.5; + } + if (typeof value === 'number') { + return Math.round(instanceVariation * dailyGrowth * value); + } + return value; +}; + +const mapValuesDeep = ( + obj: Record, + iterator: (value: unknown, key: string, obj: Record) => unknown +): Record => + mapValues(obj, (val, key) => + isPlainObject(val) ? mapValuesDeep(val, iterator) : iterator(val, key!, obj) + ); + +export async function generateSampleDocuments( + options: DeepPartial & { + collectTelemetryParams: CollectTelemetryParams; + } +) { + const { collectTelemetryParams, ...preferredOptions } = options; + + const opts: GenerateOptions = defaultsDeep( + { + days: 100, + instances: 50, + variation: { + min: 0.1, + max: 4 + } + }, + preferredOptions + ); + + const sample = await collectDataTelemetry(collectTelemetryParams); + + const dateOfScriptExecution = new Date(); + + return flatten( + range(0, opts.instances).map(instanceNo => { + const instanceId = uuid.v4(); + const defaults = { + cluster_uuid: instanceId, + stack_stats: { + kibana: { + versions: { + version: '8.0.0' + } + } + } + }; + + const instanceVariation = + Math.random() * (opts.variation.max - opts.variation.min) + + opts.variation.min; + + return range(0, opts.days).map(dayNo => { + const dailyGrowth = Math.pow(1.005, opts.days - 1 - dayNo); + + const timestamp = Date.UTC( + dateOfScriptExecution.getFullYear(), + dateOfScriptExecution.getMonth(), + -dayNo + ); + + const generated = mapValuesDeep(omit(sample, 'versions'), value => + randomize(value, instanceVariation, dailyGrowth) + ); + + return merge({}, defaults, { + timestamp, + stack_stats: { + kibana: { + plugins: { + apm: merge({}, sample, generated) + } + } + } + }); + }); + }) + ); +} diff --git a/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts new file mode 100644 index 000000000000000..0a06015110a5d19 --- /dev/null +++ b/x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/index.ts @@ -0,0 +1,199 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import fs from 'fs'; +import path from 'path'; +import Ocktokit from '@octokit/rest'; +import { merge, chunk, flatten, pick, identity } from 'lodash'; +import axios from 'axios'; +import yaml from 'js-yaml'; +import { Client } from 'elasticsearch'; +import { argv } from 'yargs'; +import { promisify } from 'util'; +import { Logger } from 'kibana/server'; +// @ts-ignore +import consoleStamp from 'console-stamp'; +import { downloadTelemetryMapping } from './download-telemetry-mapping'; +import mapping from '../../mappings.json'; +import { generateSampleDocuments } from './generate-sample-documents'; + +consoleStamp(console, '[HH:MM:ss.l]'); + +const githubToken = process.env.GITHUB_TOKEN; + +const kibanaConfigDir = path.join(__filename, '../../../../../../../config'); +const kibanaDevConfig = path.join(kibanaConfigDir, 'kibana.dev.yml'); +const kibanaConfig = path.join(kibanaConfigDir, 'kibana.yml'); + +const xpackTelemetryIndexName = 'xpack-phone-home'; + +const config = { + 'apm_oss.transactionIndices': 'apm-*', + 'apm_oss.metricsIndices': 'apm-*', + 'apm_oss.errorIndices': 'apm-*', + 'apm_oss.spanIndices': 'apm-*', + 'apm_oss.onboardingIndices': 'apm-*', + 'apm_oss.sourcemapIndices': 'apm-*', + 'elasticsearch.hosts': 'http://localhost:9200', + ...(yaml.safeLoad( + fs.readFileSync( + fs.existsSync(kibanaDevConfig) ? kibanaDevConfig : kibanaConfig, + 'utf8' + ) + ) as {}), + ...(pick( + { + 'elasticsearch.username': process.env.ELASTICSEARCH_USERNAME, + 'elasticsearch.password': process.env.ELASTICSEARCH_PASSWORD, + 'elasticsearch.hosts': process.env.ELASTICSEARCH_HOST + }, + identity + ) as { + 'elasticsearch.username': string; + 'elasticsearch.password': string; + }) +}; + +async function uploadData() { + const ocktokit = new Ocktokit({ + ...(githubToken + ? { + auth: githubToken + } + : {}) + }); + + const telemetryMapping = await downloadTelemetryMapping(ocktokit); + const kibanaMapping = mapping['apm-telemetry']; + + const httpAuth = + config['elasticsearch.username'] && config['elasticsearch.password'] + ? { + username: config['elasticsearch.username'], + password: config['elasticsearch.password'] + } + : null; + + const client = new Client({ + host: config['elasticsearch.hosts'], + ...(httpAuth + ? { + httpAuth: `${httpAuth.username}:${httpAuth.password}` + } + : {}) + }); + + if (argv.clear) { + try { + await promisify(client.indices.delete.bind(client))({ + index: xpackTelemetryIndexName + }); + } catch (err) { + // 404 = index not found, totally okay + if (err.status !== 404) { + throw err; + } + } + } + + const newMapping = { + ...telemetryMapping, + mappings: merge( + { + ...telemetryMapping.mappings.doc + }, + { + properties: { + stack_stats: { + properties: { + kibana: { + properties: { + plugins: { + properties: { + apm: kibanaMapping + } + } + } + } + } + } + } + } + ) + }; + + await axios.put( + `${config['elasticsearch.hosts']}/_template/xpack-phone-home`, + newMapping, + { + ...(httpAuth ? { auth: httpAuth } : {}) + } + ); + + const sampleDocuments = await generateSampleDocuments({ + collectTelemetryParams: { + logger: (console as unknown) as Logger, + indices: { + ...config, + apmCustomLinkIndex: '.apm-custom-links', + apmAgentConfigurationIndex: '.apm-agent-configuration' + }, + search: body => { + return promisify(client.search.bind(client))({ + ...body, + requestTimeout: 120000 + }) as any; + }, + indicesStats: body => { + return promisify(client.indices.stats.bind(client))({ + ...body, + requestTimeout: 120000 + }) as any; + } + } + }); + + const chunks = chunk(sampleDocuments, 250); + + await chunks.reduce((prev, documents) => { + return prev.then(async () => { + const body = flatten( + documents.map(doc => [{ index: { _index: 'xpack-phone-home' } }, doc]) + ); + + await promisify(client.bulk.bind(client))({ + body, + refresh: true + }); + }); + }, Promise.resolve()); +} + +uploadData() + .catch(e => { + if ('response' in e) { + if (typeof e.response === 'string') { + // eslint-disable-next-line no-console + console.log(e.response); + } else { + // eslint-disable-next-line no-console + console.log( + JSON.stringify( + e.response, + ['status', 'statusText', 'headers', 'data'], + 2 + ) + ); + } + } else { + // eslint-disable-next-line no-console + console.log(e); + } + process.exit(1); + }) + .then(() => { + // eslint-disable-next-line no-console + console.log('Finished uploading generated telemetry data'); + }); diff --git a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap index b4b4e7866e9b7d6..41d97f606e20d1b 100644 --- a/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap +++ b/x-pack/plugins/apm/common/__snapshots__/elasticsearch_fieldnames.test.ts.snap @@ -54,7 +54,7 @@ exports[`Error METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Error OBSERVER_LISTENING 1`] = `undefined`; -exports[`Error OBSERVER_VERSION_MAJOR 1`] = `undefined`; +exports[`Error OBSERVER_VERSION_MAJOR 1`] = `8`; exports[`Error PARENT_ID 1`] = `"parentId"`; @@ -166,7 +166,7 @@ exports[`Span METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Span OBSERVER_LISTENING 1`] = `undefined`; -exports[`Span OBSERVER_VERSION_MAJOR 1`] = `undefined`; +exports[`Span OBSERVER_VERSION_MAJOR 1`] = `8`; exports[`Span PARENT_ID 1`] = `"parentId"`; @@ -278,7 +278,7 @@ exports[`Transaction METRIC_SYSTEM_TOTAL_MEMORY 1`] = `undefined`; exports[`Transaction OBSERVER_LISTENING 1`] = `undefined`; -exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `undefined`; +exports[`Transaction OBSERVER_VERSION_MAJOR 1`] = `8`; exports[`Transaction PARENT_ID 1`] = `"parentId"`; diff --git a/x-pack/plugins/apm/common/agent_name.ts b/x-pack/plugins/apm/common/agent_name.ts index bb68eb88b8e1889..86123768d42dbe5 100644 --- a/x-pack/plugins/apm/common/agent_name.ts +++ b/x-pack/plugins/apm/common/agent_name.ts @@ -4,36 +4,40 @@ * you may not use this file except in compliance with the Elastic License. */ +import { AgentName } from '../typings/es_schemas/ui/fields/agent'; + /* * Agent names can be any string. This list only defines the official agents * that we might want to target specifically eg. linking to their documentation * & telemetry reporting. Support additional agent types by appending * definitions in mappings.json (for telemetry), the AgentName type, and the - * agentNames object. + * AGENT_NAMES array. */ -import { AgentName } from '../typings/es_schemas/ui/fields/agent'; -const agentNames: { [agentName in AgentName]: agentName } = { - python: 'python', - java: 'java', - nodejs: 'nodejs', - 'js-base': 'js-base', - 'rum-js': 'rum-js', - dotnet: 'dotnet', - ruby: 'ruby', - go: 'go' -}; +export const AGENT_NAMES: AgentName[] = [ + 'java', + 'js-base', + 'rum-js', + 'dotnet', + 'go', + 'java', + 'nodejs', + 'python', + 'ruby' +]; export function isAgentName(agentName: string): boolean { - return Object.values(agentNames).includes(agentName as AgentName); + return AGENT_NAMES.includes(agentName as AgentName); } -export function isRumAgentName(agentName: string | undefined) { - return ( - agentName === agentNames['js-base'] || agentName === agentNames['rum-js'] - ); +export function isRumAgentName( + agentName: string | undefined +): agentName is AgentName { + return agentName === 'js-base' || agentName === 'rum-js'; } -export function isJavaAgentName(agentName: string | undefined) { - return agentName === agentNames.java; +export function isJavaAgentName( + agentName: string | undefined +): agentName is 'java' { + return agentName === 'java'; } diff --git a/x-pack/plugins/apm/common/apm_saved_object_constants.ts b/x-pack/plugins/apm/common/apm_saved_object_constants.ts index ac43b700117c62a..0529d90fe940a45 100644 --- a/x-pack/plugins/apm/common/apm_saved_object_constants.ts +++ b/x-pack/plugins/apm/common/apm_saved_object_constants.ts @@ -4,11 +4,13 @@ * you may not use this file except in compliance with the Elastic License. */ -// APM Services telemetry -export const APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE = - 'apm-services-telemetry'; -export const APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID = 'apm-services-telemetry'; +// the types have to match the names of the saved object mappings +// in /x-pack/legacy/plugins/apm/mappings.json // APM indices export const APM_INDICES_SAVED_OBJECT_TYPE = 'apm-indices'; export const APM_INDICES_SAVED_OBJECT_ID = 'apm-indices'; + +// APM telemetry +export const APM_TELEMETRY_SAVED_OBJECT_TYPE = 'apm-telemetry'; +export const APM_TELEMETRY_SAVED_OBJECT_ID = 'apm-telemetry'; diff --git a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts index 1add2427d16a022..63fa749cd9f2cc6 100644 --- a/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts +++ b/x-pack/plugins/apm/common/elasticsearch_fieldnames.test.ts @@ -15,7 +15,10 @@ describe('Transaction', () => { const transaction: AllowUnknownProperties = { '@timestamp': new Date().toString(), '@metadata': 'whatever', - observer: 'whatever', + observer: { + version: 'whatever', + version_major: 8 + }, agent: { name: 'java', version: 'agent version' @@ -63,7 +66,10 @@ describe('Span', () => { const span: AllowUnknownProperties = { '@timestamp': new Date().toString(), '@metadata': 'whatever', - observer: 'whatever', + observer: { + version: 'whatever', + version_major: 8 + }, agent: { name: 'java', version: 'agent version' @@ -107,7 +113,10 @@ describe('Span', () => { describe('Error', () => { const errorDoc: AllowUnknownProperties = { '@metadata': 'whatever', - observer: 'whatever', + observer: { + version: 'whatever', + version_major: 8 + }, agent: { name: 'java', version: 'agent version' diff --git a/x-pack/plugins/apm/kibana.json b/x-pack/plugins/apm/kibana.json index 96579377c95e8ff..80916f8d1e72f05 100644 --- a/x-pack/plugins/apm/kibana.json +++ b/x-pack/plugins/apm/kibana.json @@ -3,7 +3,10 @@ "server": true, "version": "8.0.0", "kibanaVersion": "kibana", - "configPath": ["xpack", "apm"], + "configPath": [ + "xpack", + "apm" + ], "ui": false, "requiredPlugins": ["apm_oss", "data", "home", "licensing"], "optionalPlugins": ["cloud", "usageCollection"] diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts deleted file mode 100644 index c45c74a791aeebd..000000000000000 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/__test__/index.test.ts +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License; - * you may not use this file except in compliance with the Elastic License. - */ - -import { SavedObjectAttributes } from '../../../../../../../src/core/server'; -import { createApmTelementry, storeApmServicesTelemetry } from '../index'; -import { - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, - APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID -} from '../../../../common/apm_saved_object_constants'; - -describe('apm_telemetry', () => { - describe('createApmTelementry', () => { - it('should create a ApmTelemetry object with boolean flag and frequency map of the given list of AgentNames', () => { - const apmTelemetry = createApmTelementry([ - 'go', - 'nodejs', - 'go', - 'js-base' - ]); - expect(apmTelemetry.has_any_services).toBe(true); - expect(apmTelemetry.services_per_agent).toMatchObject({ - go: 2, - nodejs: 1, - 'js-base': 1 - }); - }); - it('should ignore undefined or unknown AgentName values', () => { - const apmTelemetry = createApmTelementry([ - 'go', - 'nodejs', - 'go', - 'js-base', - 'example-platform' as any, - undefined as any - ]); - expect(apmTelemetry.services_per_agent).toMatchObject({ - go: 2, - nodejs: 1, - 'js-base': 1 - }); - }); - }); - - describe('storeApmServicesTelemetry', () => { - let apmTelemetry: SavedObjectAttributes; - let savedObjectsClient: any; - - beforeEach(() => { - apmTelemetry = { - has_any_services: true, - services_per_agent: { - go: 2, - nodejs: 1, - 'js-base': 1 - } - }; - savedObjectsClient = { create: jest.fn() }; - }); - - it('should call savedObjectsClient create with the given ApmTelemetry object', () => { - storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); - expect(savedObjectsClient.create.mock.calls[0][1]).toBe(apmTelemetry); - }); - - it('should call savedObjectsClient create with the apm-telemetry document type and ID', () => { - storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); - expect(savedObjectsClient.create.mock.calls[0][0]).toBe( - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE - ); - expect(savedObjectsClient.create.mock.calls[0][2].id).toBe( - APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID - ); - }); - - it('should call savedObjectsClient create with overwrite: true', () => { - storeApmServicesTelemetry(savedObjectsClient, apmTelemetry); - expect(savedObjectsClient.create.mock.calls[0][2].overwrite).toBe(true); - }); - }); -}); diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts new file mode 100644 index 000000000000000..0cd81f85dd249e0 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/index.ts @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { merge } from 'lodash'; +import { Logger, CallAPIOptions } from 'kibana/server'; +import { IndicesStatsParams, Client } from 'elasticsearch'; +import { + ESSearchRequest, + ESSearchResponse +} from '../../../../typings/elasticsearch'; +import { ApmIndicesConfig } from '../../settings/apm_indices/get_apm_indices'; +import { tasks } from './tasks'; +import { APMDataTelemetry } from '../types'; + +type TelemetryTaskExecutor = (params: { + indices: ApmIndicesConfig; + search( + params: TSearchRequest + ): Promise>; + indicesStats( + params: IndicesStatsParams, + options?: CallAPIOptions + ): ReturnType; +}) => Promise; + +export interface TelemetryTask { + name: string; + executor: TelemetryTaskExecutor; +} + +export type CollectTelemetryParams = Parameters[0] & { + logger: Logger; +}; + +export function collectDataTelemetry({ + search, + indices, + logger, + indicesStats +}: CollectTelemetryParams) { + return tasks.reduce((prev, task) => { + return prev.then(async data => { + logger.info(`Executing APM telemetry task ${task.name}`); + try { + const time = process.hrtime(); + const next = await task.executor({ + search, + indices, + indicesStats + }); + const took = process.hrtime(time); + + return merge({}, data, next, { + tasks: { + [task.name]: { + took: { + ms: Math.round(took[0] * 1000 + took[1] / 1e6) + } + } + } + }); + } catch (err) { + logger.warn(`Failed executing APM telemetry task ${task.name}`); + logger.info(err); + return data; + } + }); + }, Promise.resolve({} as APMDataTelemetry)); +} diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts new file mode 100644 index 000000000000000..1b9209e1429d4ed --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/collect_data_telemetry/tasks.ts @@ -0,0 +1,461 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ +import { flatten, merge, set, sum } from 'lodash'; +import { AgentName } from '../../../../typings/es_schemas/ui/fields/agent'; +import { AGENT_NAMES } from '../../../../common/agent_name'; +import { Transaction } from '../../../../typings/es_schemas/ui/transaction'; +import { + PROCESSOR_EVENT, + SERVICE_NAME, + SERVICE_AGENT_NAME, + ERROR_GROUP_ID, + TRANSACTION_NAME, + PARENT_ID +} from '../../../../common/elasticsearch_fieldnames'; +import { Span } from '../../../../typings/es_schemas/ui/span'; +import { APMError } from '../../../../typings/es_schemas/ui/apm_error'; +import { TelemetryTask } from '.'; +import { APMTelemetry } from '../types'; + +const TIME_RANGES = ['1d', 'all'] as const; +type TimeRange = typeof TIME_RANGES[number]; + +export const tasks: TelemetryTask[] = [ + { + name: 'processor_events', + executor: async ({ indices, search }) => { + const indicesByProcessorEvent = { + error: indices['apm_oss.errorIndices'], + metric: indices['apm_oss.metricsIndices'], + span: indices['apm_oss.spanIndices'], + transaction: indices['apm_oss.transactionIndices'], + onboarding: indices['apm_oss.onboardingIndices'], + sourcemap: indices['apm_oss.sourcemapIndices'] + }; + + type ProcessorEvent = keyof typeof indicesByProcessorEvent; + + const jobs: Array<{ + processorEvent: ProcessorEvent; + timeRange: TimeRange; + }> = flatten( + (Object.keys( + indicesByProcessorEvent + ) as ProcessorEvent[]).map(processorEvent => + TIME_RANGES.map(timeRange => ({ processorEvent, timeRange })) + ) + ); + + const counts = await jobs.reduce((prevJob, current) => { + return prevJob.then(async data => { + const { processorEvent, timeRange } = current; + + return merge({}, data, { + [processorEvent]: { + [timeRange]: ( + await search({ + index: indicesByProcessorEvent[processorEvent], + body: { + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: processorEvent } }, + ...(timeRange !== 'all' + ? [ + { + range: { + '@timestamp': { + gte: `now-${timeRange}` + } + } + } + ] + : []) + ] + } + }, + track_total_hits: true + } + }) + ).hits.total.value + } + }); + }); + }, Promise.resolve({} as Record>)); + + return { + counts + }; + } + }, + { + name: 'agent_configuration', + executor: async ({ indices, search }) => { + const agentConfigurationCount = ( + await search({ + index: indices.apmAgentConfigurationIndex, + body: { + size: 0, + track_total_hits: true + } + }) + ).hits.total.value; + + return { + counts: { + agent_configuration: { + all: agentConfigurationCount + } + } + }; + } + }, + { + name: 'services', + executor: async ({ indices, search }) => { + const servicesPerAgent = await AGENT_NAMES.reduce( + (prevJob, agentName) => { + return prevJob.then(async data => { + const response = await search({ + index: [ + indices['apm_oss.errorIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.transactionIndices'] + ], + body: { + query: { + bool: { + filter: { + term: { + [SERVICE_AGENT_NAME]: agentName + } + } + } + }, + aggs: { + services: { + cardinality: { + field: SERVICE_NAME + } + } + } + } + }); + + return { + ...data, + [agentName]: response.aggregations?.services.value || 0 + }; + }); + }, + Promise.resolve({} as Record) + ); + + return { + has_any_services: sum(Object.values(servicesPerAgent)) > 0, + services_per_agent: servicesPerAgent + }; + } + }, + { + name: 'versions', + executor: async ({ search, indices }) => { + const response = await search({ + index: [ + indices['apm_oss.transactionIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.errorIndices'] + ], + body: { + query: { + exists: { + field: 'observer.version' + } + }, + terminate_after: 1, + size: 1, + sort: { + '@timestamp': 'desc' + } + } + }); + + const hit = response.hits.hits[0]?._source as Pick< + Transaction | Span | APMError, + 'observer' + >; + + if (!hit || !hit.observer?.version) { + return {}; + } + + const [major, minor, patch] = hit.observer.version + .split('.') + .map(part => Number(part)); + + return { + versions: { + apm_server: { + major, + minor, + patch + } + } + }; + } + }, + { + name: 'groupings', + executor: async ({ search, indices }) => { + const range1d = { range: { '@timestamp': { gte: 'now-1d' } } }; + const errorGroupsCount = ( + await search({ + index: indices['apm_oss.errorIndices'], + body: { + size: 0, + query: { + bool: { + filter: [{ term: { [PROCESSOR_EVENT]: 'error' } }, range1d] + } + }, + aggs: { + top_service: { + terms: { + field: SERVICE_NAME, + order: { + error_groups: 'desc' + }, + size: 1 + }, + aggs: { + error_groups: { + cardinality: { + field: ERROR_GROUP_ID + } + } + } + } + } + } + }) + ).aggregations?.top_service.buckets[0]?.error_groups.value; + + const transactionGroupsCount = ( + await search({ + index: indices['apm_oss.transactionIndices'], + body: { + size: 0, + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: 'transaction' } }, + range1d + ] + } + }, + aggs: { + top_service: { + terms: { + field: SERVICE_NAME, + order: { + transaction_groups: 'desc' + }, + size: 1 + }, + aggs: { + transaction_groups: { + cardinality: { + field: TRANSACTION_NAME + } + } + } + } + } + } + }) + ).aggregations?.top_service.buckets[0]?.transaction_groups.value; + + const tracesPerDayCount = ( + await search({ + index: indices['apm_oss.transactionIndices'], + body: { + query: { + bool: { + filter: [ + { term: { [PROCESSOR_EVENT]: 'transaction' } }, + range1d + ], + must_not: { + exists: { field: PARENT_ID } + } + } + }, + track_total_hits: true, + size: 0 + } + }) + ).hits.total.value; + + const servicesCount = ( + await search({ + index: [ + indices['apm_oss.transactionIndices'], + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'] + ], + body: { + query: { + bool: { + filter: [range1d] + } + }, + aggs: { + service_name: { + cardinality: { + field: SERVICE_NAME + } + } + } + } + }) + ).aggregations?.service_name.value; + + return { + counts: { + max_error_groups_per_service: { + '1d': errorGroupsCount || 0 + }, + max_transaction_groups_per_service: { + '1d': transactionGroupsCount || 0 + }, + traces: { + '1d': tracesPerDayCount || 0 + }, + services: { + '1d': servicesCount || 0 + } + } + }; + } + }, + { + name: 'integrations', + executor: async ({ search }) => { + const apmJobs = ['high_mean_response_time']; + + const response = await search({ + index: apmJobs.map(job => `.ml-anomalies-*-${job}`), + body: { + size: 0 + } + }); + + return { + integrations: { + ml: { + has_anomalies_indices: response.hits.total.value > 0 + } + } + }; + } + }, + { + name: 'agents', + executor: async ({ search, indices }) => { + const agentFields = [ + 'agent.version', + 'service.framework.name', + 'service.framework.version', + 'service.language.name', + 'service.language.version', + 'service.runtime.name', + 'service.runtime.version' + ] as const; + + const jobs = flatten( + AGENT_NAMES.map(agent => agentFields.map(field => ({ agent, field }))) + ); + + const agentData = await jobs.reduce((prevJob, job) => { + return prevJob.then(async data => { + const { field, agent } = job; + + const response = await search({ + index: [ + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.transactionIndices'] + ], + body: { + size: 3, + query: { + bool: { + filter: [ + { term: { [SERVICE_AGENT_NAME]: agent } }, + { exists: { field } } + ] + } + }, + collapse: { + field + }, + terminate_after: 3, + sort: { + '@timestamp': 'desc' + } + } + }); + const values = response.hits.hits.map(hit => { + return hit.fields[field][0]; + }); + + set(data, `${agent}.${job.field}`, values); + + return data; + }); + }, Promise.resolve({} as APMTelemetry['agents'])); + + return { + agents: agentData + }; + } + }, + { + name: 'indices_stats', + executor: async ({ indicesStats, indices }) => { + const response = await indicesStats({ + index: [ + indices.apmAgentConfigurationIndex, + indices['apm_oss.errorIndices'], + indices['apm_oss.metricsIndices'], + indices['apm_oss.onboardingIndices'], + indices['apm_oss.sourcemapIndices'], + indices['apm_oss.spanIndices'], + indices['apm_oss.transactionIndices'] + ] + }); + + return { + indices: { + shards: { + total: response._shards.total + }, + all: { + total: { + docs: { + count: response._all.total.docs.count + }, + store: { + size_in_bytes: response._all.total.store.size_in_bytes + } + } + } + } + }; + } + } +]; diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts index a2b04947308267b..10fd9ba4e30db03 100644 --- a/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/index.ts @@ -4,59 +4,116 @@ * you may not use this file except in compliance with the Elastic License. */ -import { countBy } from 'lodash'; -import { SavedObjectAttributes } from '../../../../../../src/core/server'; -import { isAgentName } from '../../../common/agent_name'; +import { CoreSetup, Logger } from 'src/core/server'; +import { Observable } from 'rxjs'; +import { take } from 'rxjs/operators'; +import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; import { - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, - APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID + TaskManagerStartContract, + TaskManagerSetupContract +} from '../../../../task_manager/server'; +import { APMLegacyServer } from '../../routes/typings'; +import { getApmIndices } from '../settings/apm_indices/get_apm_indices'; +import { + APM_TELEMETRY_SAVED_OBJECT_ID, + APM_TELEMETRY_SAVED_OBJECT_TYPE } from '../../../common/apm_saved_object_constants'; -import { UsageCollectionSetup } from '../../../../../../src/plugins/usage_collection/server'; -import { InternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; - -export function createApmTelementry( - agentNames: string[] = [] -): SavedObjectAttributes { - const validAgentNames = agentNames.filter(isAgentName); - return { - has_any_services: validAgentNames.length > 0, - services_per_agent: countBy(validAgentNames) +import { + collectDataTelemetry, + CollectTelemetryParams +} from './collect_data_telemetry'; +import { APMConfig } from '../..'; +import { getInternalSavedObjectsClient } from '../helpers/get_internal_saved_objects_client'; + +const APM_TELEMETRY_TASK_NAME = 'apm-telemetry-task'; + +export async function createApmTelemetry({ + core, + config$, + server, + usageCollector, + taskManager, + logger +}: { + core: CoreSetup; + config$: Observable; + server: APMLegacyServer; + usageCollector: UsageCollectionSetup; + taskManager: TaskManagerSetupContract; + logger: Logger; +}) { + const savedObjectsClient = await getInternalSavedObjectsClient(core); + + const collectAndStore = async () => { + const config = await config$.pipe(take(1)).toPromise(); + const esClient = core.elasticsearch.dataClient; + + const indices = await getApmIndices({ + config, + savedObjectsClient + }); + + const search = esClient.callAsInternalUser.bind( + null, + 'search' + ) as CollectTelemetryParams['search']; + + const indicesStats = esClient.callAsInternalUser.bind( + null, + 'indices.stats' + ) as CollectTelemetryParams['indicesStats']; + + const dataTelemetry = await collectDataTelemetry({ + search, + indices, + logger, + indicesStats + }); + + await savedObjectsClient.create( + APM_TELEMETRY_SAVED_OBJECT_TYPE, + dataTelemetry, + { id: APM_TELEMETRY_SAVED_OBJECT_TYPE, overwrite: true } + ); }; -} -export async function storeApmServicesTelemetry( - savedObjectsClient: InternalSavedObjectsClient, - apmTelemetry: SavedObjectAttributes -) { - return savedObjectsClient.create( - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, - apmTelemetry, - { - id: APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID, - overwrite: true + taskManager.registerTaskDefinitions({ + [APM_TELEMETRY_TASK_NAME]: { + title: 'Collect APM telemetry', + type: APM_TELEMETRY_TASK_NAME, + createTaskRunner: () => { + return { + run: async () => { + await collectAndStore(); + } + }; + } } - ); -} + }); -export function makeApmUsageCollector( - usageCollector: UsageCollectionSetup, - savedObjectsRepository: InternalSavedObjectsClient -) { - const apmUsageCollector = usageCollector.makeUsageCollector({ + usageCollector.makeUsageCollector({ type: 'apm', - fetch: async () => { - try { - const apmTelemetrySavedObject = await savedObjectsRepository.get( - APM_SERVICES_TELEMETRY_SAVED_OBJECT_TYPE, - APM_SERVICES_TELEMETRY_SAVED_OBJECT_ID - ); - return apmTelemetrySavedObject.attributes; - } catch (err) { - return createApmTelementry(); - } + fetch: () => { + return savedObjectsClient.get( + APM_TELEMETRY_SAVED_OBJECT_TYPE, + APM_TELEMETRY_SAVED_OBJECT_ID + ); }, isReady: () => true }); +} - usageCollector.registerCollector(apmUsageCollector); +export function scheduleApmTelemetryTasks( + taskManager: TaskManagerStartContract +) { + taskManager.ensureScheduled({ + id: APM_TELEMETRY_TASK_NAME, + taskType: APM_TELEMETRY_TASK_NAME, + schedule: { + interval: '1m' + }, + scope: ['apm'], + params: {}, + state: {} + }); } diff --git a/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts new file mode 100644 index 000000000000000..aee3a23c00c2e91 --- /dev/null +++ b/x-pack/plugins/apm/server/lib/apm_telemetry/types.ts @@ -0,0 +1,96 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +import { DeepPartial } from 'utility-types'; +import { AgentName } from '../../../typings/es_schemas/ui/fields/agent'; + +export interface TimeframeMap { + '1d': number; + all: number; +} + +export type TimeframeMap1d = Pick; +export type TimeframeMapAll = Pick; + +export type APMDataTelemetry = DeepPartial<{ + has_any_services: boolean; + services_per_agent: Record; + versions: { + apm_server: { + minor: number; + major: number; + patch: number; + }; + }; + counts: { + transaction: TimeframeMap; + span: TimeframeMap; + error: TimeframeMap; + metric: TimeframeMap; + sourcemap: TimeframeMap; + onboarding: TimeframeMap; + agent_configuration: TimeframeMapAll; + max_transaction_groups_per_service: TimeframeMap; + max_error_groups_per_service: TimeframeMap; + traces: TimeframeMap; + services: TimeframeMap; + }; + integrations: { + ml: { + has_anomalies_indices: boolean; + }; + }; + agents: Record< + AgentName, + { + agent: { + version: string[]; + }; + service: { + framework: { + name: string[]; + version: string[]; + }; + language: { + name: string[]; + version: string[]; + }; + runtime: { + name: string[]; + version: string[]; + }; + }; + } + >; + indices: { + shards: { + total: number; + }; + all: { + total: { + docs: { + count: number; + }; + store: { + size_in_bytes: number; + }; + }; + }; + }; + tasks: Record< + | 'processor_events' + | 'agent_configuration' + | 'services' + | 'versions' + | 'groupings' + | 'integrations' + | 'agents' + | 'indices_stats', + { took: { ms: number } } + >; +}>; + +export type APMTelemetry = APMDataTelemetry; diff --git a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts index 40a2a0e7216a099..8e8cf698a84cfac 100644 --- a/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts +++ b/x-pack/plugins/apm/server/lib/helpers/setup_request.test.ts @@ -39,6 +39,19 @@ function getMockRequest() { _debug: false } }, + __LEGACY: { + server: { + plugins: { + elasticsearch: { + getCluster: jest.fn().mockReturnValue({ callWithInternalUser: {} }) + } + }, + savedObjects: { + SavedObjectsClient: jest.fn(), + getSavedObjectsRepository: jest.fn() + } + } + }, core: { elasticsearch: { dataClient: { diff --git a/x-pack/plugins/apm/server/plugin.ts b/x-pack/plugins/apm/server/plugin.ts index db14730f802a96f..6d283a15739e538 100644 --- a/x-pack/plugins/apm/server/plugin.ts +++ b/x-pack/plugins/apm/server/plugin.ts @@ -3,14 +3,22 @@ * or more contributor license agreements. Licensed under the Elastic License; * you may not use this file except in compliance with the Elastic License. */ -import { PluginInitializerContext, Plugin, CoreSetup } from 'src/core/server'; +import { + PluginInitializerContext, + Plugin, + CoreSetup, + CoreStart +} from 'src/core/server'; import { Observable, combineLatest, AsyncSubject } from 'rxjs'; import { map, take } from 'rxjs/operators'; import { Server } from 'hapi'; import { once } from 'lodash'; -import { UsageCollectionSetup } from 'src/plugins/usage_collection/server'; +import { UsageCollectionSetup } from '../../../../src/plugins/usage_collection/server'; +import { + TaskManagerSetupContract, + TaskManagerStartContract +} from '../../task_manager/server'; import { APMOSSPluginSetup } from '../../../../src/plugins/apm_oss/server'; -import { makeApmUsageCollector } from './lib/apm_telemetry'; import { createApmAgentConfigurationIndex } from './lib/settings/agent_configuration/create_agent_config_index'; import { createApmCustomLinkIndex } from './lib/settings/custom_link/create_custom_link_index'; import { createApmApi } from './routes/create_apm_api'; @@ -21,6 +29,10 @@ import { tutorialProvider } from './tutorial'; import { CloudSetup } from '../../cloud/server'; import { getInternalSavedObjectsClient } from './lib/helpers/get_internal_saved_objects_client'; import { LicensingPluginSetup } from '../../licensing/public'; +import { + createApmTelemetry, + scheduleApmTelemetryTasks +} from './lib/apm_telemetry'; export interface LegacySetup { server: Server; @@ -47,6 +59,7 @@ export class APMPlugin implements Plugin { licensing: LicensingPluginSetup; cloud?: CloudSetup; usageCollection?: UsageCollectionSetup; + taskManager?: TaskManagerSetupContract; } ) { const logger = this.initContext.logger.get('apm'); @@ -56,6 +69,17 @@ export class APMPlugin implements Plugin { ); this.legacySetup$.subscribe(__LEGACY => { + if (plugins.taskManager && plugins.usageCollection) { + createApmTelemetry({ + core, + config$: mergedConfig$, + server: __LEGACY.server, + usageCollector: plugins.usageCollection, + taskManager: plugins.taskManager, + logger + }); + } + createApmApi().init(core, { config$: mergedConfig$, logger, __LEGACY }); }); @@ -89,18 +113,6 @@ export class APMPlugin implements Plugin { }) ); - const usageCollection = plugins.usageCollection; - if (usageCollection) { - getInternalSavedObjectsClient(core) - .then(savedObjectsClient => { - makeApmUsageCollector(usageCollection, savedObjectsClient); - }) - .catch(error => { - logger.error('Unable to initialize use collection'); - logger.error(error.message); - }); - } - return { config$: mergedConfig$, registerLegacyAPI: once((__LEGACY: LegacySetup) => { @@ -115,6 +127,16 @@ export class APMPlugin implements Plugin { }; } - public start() {} + public async start( + core: CoreStart, + plugins: { + taskManager?: TaskManagerStartContract; + } + ) { + if (plugins.taskManager) { + scheduleApmTelemetryTasks(plugins.taskManager); + } + } + public stop() {} } diff --git a/x-pack/plugins/apm/server/routes/create_api/index.test.ts b/x-pack/plugins/apm/server/routes/create_api/index.test.ts index e639bb5101e2fd5..312dae1d1f9d245 100644 --- a/x-pack/plugins/apm/server/routes/create_api/index.test.ts +++ b/x-pack/plugins/apm/server/routes/create_api/index.test.ts @@ -36,6 +36,7 @@ const getCoreMock = () => { put, createRouter, context: { + measure: () => undefined, config$: new BehaviorSubject({} as APMConfig), logger: ({ error: jest.fn() diff --git a/x-pack/plugins/apm/server/routes/services.ts b/x-pack/plugins/apm/server/routes/services.ts index 2d4fae9d2707a5c..1c6561ee24c9372 100644 --- a/x-pack/plugins/apm/server/routes/services.ts +++ b/x-pack/plugins/apm/server/routes/services.ts @@ -5,11 +5,6 @@ */ import * as t from 'io-ts'; -import { AgentName } from '../../typings/es_schemas/ui/fields/agent'; -import { - createApmTelementry, - storeApmServicesTelemetry -} from '../lib/apm_telemetry'; import { setupRequest } from '../lib/helpers/setup_request'; import { getServiceAgentName } from '../lib/services/get_service_agent_name'; import { getServices } from '../lib/services/get_services'; @@ -18,7 +13,6 @@ import { getServiceNodeMetadata } from '../lib/services/get_service_node_metadat import { createRoute } from './create_route'; import { uiFiltersRt, rangeRt } from './default_api_types'; import { getServiceAnnotations } from '../lib/services/annotations'; -import { getInternalSavedObjectsClient } from '../lib/helpers/get_internal_saved_objects_client'; export const servicesRoute = createRoute(core => ({ path: '/api/apm/services', @@ -29,16 +23,6 @@ export const servicesRoute = createRoute(core => ({ const setup = await setupRequest(context, request); const services = await getServices(setup); - // Store telemetry data derived from services - const agentNames = services.items.map( - ({ agentName }) => agentName as AgentName - ); - const apmTelemetry = createApmTelementry(agentNames); - const savedObjectsClient = await getInternalSavedObjectsClient(core); - storeApmServicesTelemetry(savedObjectsClient, apmTelemetry).catch(error => { - context.logger.error(error.message); - }); - return services; } })); diff --git a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts index 6d3620f11a87b23..8a8d256cf427323 100644 --- a/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts +++ b/x-pack/plugins/apm/typings/elasticsearch/aggregations.ts @@ -126,6 +126,16 @@ export interface AggregationOptionsByType { combine_script: Script; reduce_script: Script; }; + date_range: { + field: string; + format?: string; + ranges: Array< + | { from: string | number } + | { to: string | number } + | { from: string | number; to: string | number } + >; + keyed?: boolean; + }; } type AggregationType = keyof AggregationOptionsByType; @@ -136,6 +146,15 @@ type AggregationOptionsMap = Unionize< } > & { aggs?: AggregationInputMap }; +interface DateRangeBucket { + key: string; + to?: number; + from?: number; + to_as_string?: string; + from_as_string?: string; + doc_count: number; +} + export interface AggregationInputMap { [key: string]: AggregationOptionsMap; } @@ -276,6 +295,11 @@ interface AggregationResponsePart< scripted_metric: { value: unknown; }; + date_range: { + buckets: TAggregationOptionsMap extends { date_range: { keyed: true } } + ? Record + : { buckets: DateRangeBucket[] }; + }; } // Type for debugging purposes. If you see an error in AggregationResponseMap @@ -285,7 +309,7 @@ interface AggregationResponsePart< // type MissingAggregationResponseTypes = Exclude< // AggregationType, -// keyof AggregationResponsePart<{}> +// keyof AggregationResponsePart<{}, unknown> // >; export type AggregationResponseMap< diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts index daf65e44980b60d..8e49d02beb908b9 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/error_raw.ts @@ -15,6 +15,7 @@ import { Service } from './fields/service'; import { IStackframe } from './fields/stackframe'; import { Url } from './fields/url'; import { User } from './fields/user'; +import { Observer } from './fields/observer'; interface Processor { name: 'error'; @@ -61,4 +62,5 @@ export interface ErrorRaw extends APMBaseDoc { service: Service; url?: Url; user?: User; + observer?: Observer; } diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts b/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts new file mode 100644 index 000000000000000..42843130ec47fb3 --- /dev/null +++ b/x-pack/plugins/apm/typings/es_schemas/raw/fields/observer.ts @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +export interface Observer { + version: string; + version_major: number; +} diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts index dbd9e7ede4256b0..4d5d2c5c4a12e89 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/span_raw.ts @@ -6,6 +6,7 @@ import { APMBaseDoc } from './apm_base_doc'; import { IStackframe } from './fields/stackframe'; +import { Observer } from './fields/observer'; interface Processor { name: 'transaction'; @@ -50,4 +51,5 @@ export interface SpanRaw extends APMBaseDoc { transaction?: { id: string; }; + observer?: Observer; } diff --git a/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts b/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts index 3673f1f13c403f7..b8ebb4cf8da51d3 100644 --- a/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts +++ b/x-pack/plugins/apm/typings/es_schemas/raw/transaction_raw.ts @@ -15,6 +15,7 @@ import { Service } from './fields/service'; import { Url } from './fields/url'; import { User } from './fields/user'; import { UserAgent } from './fields/user_agent'; +import { Observer } from './fields/observer'; interface Processor { name: 'transaction'; @@ -61,4 +62,5 @@ export interface TransactionRaw extends APMBaseDoc { url?: Url; user?: User; user_agent?: UserAgent; + observer?: Observer; }