[APM] Collect telemetry about data/API performance

Closes elastic#50757.

x-pack/legacy/plugins/apm/index.ts

@@ -14,7 +14,13 @@ import mappings from './mappings.json';
 
 export const apm: LegacyPluginInitializer = kibana => {
   return new kibana.Plugin({
-    require: ['kibana', 'elasticsearch', 'xpack_main', 'apm_oss'],
+    require: [
+      'kibana',
+      'elasticsearch',
+      'xpack_main',
+      'apm_oss',
+      'task_manager'
+    ],
     id: 'apm',
     configPrefix: 'xpack.apm',
     publicDir: resolve(__dirname, 'public'),
@@ -107,10 +113,12 @@ export const apm: LegacyPluginInitializer = kibana => {
           }
         }
       });
-
       const apmPlugin = server.newPlatform.setup.plugins
         .apm as APMPluginContract;
-      apmPlugin.registerLegacyAPI({ server });
+
+      apmPlugin.registerLegacyAPI({
+        server
+      });
     }
   });
 };

x-pack/legacy/plugins/apm/mappings.json

@@ -1,46 +1,7 @@
 {
-  "apm-services-telemetry": {
-    "properties": {
-      "has_any_services": {
-        "type": "boolean"
-      },
-      "services_per_agent": {
-        "properties": {
-          "python": {
-            "type": "long",
-            "null_value": 0
-          },
-          "java": {
-            "type": "long",
-            "null_value": 0
-          },
-          "nodejs": {
-            "type": "long",
-            "null_value": 0
-          },
-          "js-base": {
-            "type": "long",
-            "null_value": 0
-          },
-          "rum-js": {
-            "type": "long",
-            "null_value": 0
-          },
-          "dotnet": {
-            "type": "long",
-            "null_value": 0
-          },
-          "ruby": {
-            "type": "long",
-            "null_value": 0
-          },
-          "go": {
-            "type": "long",
-            "null_value": 0
-          }
-        }
-      }
-    }
+  "apm-telemetry": {
+    "properties": {},
+    "dynamic": true
   },
   "apm-indices": {
     "properties": {

x-pack/legacy/plugins/apm/scripts/.gitignore

@@ -0,0 +1 @@
+yarn.lock

x-pack/legacy/plugins/apm/scripts/package.json

@@ -0,0 +1,10 @@
+{
+  "name": "apm-scripts",
+  "version": "1.0.0",
+  "main": "index.js",
+  "license": "MIT",
+  "dependencies": {
+    "@octokit/rest": "^16.35.0",
+    "console-stamp": "^0.2.9"
+  }
+}

x-pack/legacy/plugins/apm/scripts/setup-kibana-security.js

@@ -16,6 +16,7 @@
  ******************************/
 
 // compile typescript on the fly
+// eslint-disable-next-line import/no-extraneous-dependencies
 require('@babel/register')({
   extensions: ['.ts'],
   plugins: ['@babel/plugin-proposal-optional-chaining'],

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data.js

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+// compile typescript on the fly
+// eslint-disable-next-line import/no-extraneous-dependencies
+require('@babel/register')({
+  extensions: ['.ts'],
+  plugins: [
+    '@babel/plugin-proposal-optional-chaining',
+    '@babel/plugin-proposal-nullish-coalescing-operator'
+  ],
+  presets: [
+    '@babel/typescript',
+    ['@babel/preset-env', { targets: { node: 'current' } }]
+  ]
+});
+
+require('./upload-telemetry-data/index.ts');

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/download-telemetry-mapping.ts

@@ -0,0 +1,24 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import Ocktokit from '@octokit/rest';
+
+export async function downloadTelemetryMapping(ocktokit: Ocktokit) {
+  const file = await ocktokit.repos.getContents({
+    owner: 'elastic',
+    repo: 'telemetry',
+    path: 'config/templates/xpack-phone-home.json',
+    mediaType: {
+      format: 'application/vnd.github.VERSION.raw'
+    }
+  });
+
+  if (Array.isArray(file.data)) {
+    throw new Error('Expected single response, got array');
+  }
+
+  return JSON.parse(Buffer.from(file.data.content!, 'base64').toString());
+}

x-pack/legacy/plugins/apm/scripts/upload-telemetry-data/generate-sample-documents.ts

@@ -0,0 +1,120 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+import { DeepPartial } from 'utility-types';
+import {
+  merge,
+  omit,
+  defaultsDeep,
+  range,
+  mapValues,
+  isPlainObject,
+  flatten
+} from 'lodash';
+import uuid from 'uuid';
+import {
+  CollectTelemetryParams,
+  collectDataTelemetry
+} from '../../../../../plugins/apm/server/lib/apm_telemetry/collect_data_telemetry';
+
+interface GenerateOptions {
+  days: number;
+  instances: number;
+  variation: {
+    min: number;
+    max: number;
+  };
+}
+
+const randomize = (
+  value: unknown,
+  instanceVariation: number,
+  dailyGrowth: number
+) => {
+  if (typeof value === 'boolean') {
+    return Math.random() > 0.5;
+  }
+  if (typeof value === 'number') {
+    return Math.round(instanceVariation * dailyGrowth * value);
+  }
+  return value;
+};
+
+const mapValuesDeep = (
+  obj: Record<string, any>,
+  iterator: (value: unknown, key: string, obj: Record<string, any>) => unknown
+): Record<string, any> =>
+  mapValues(obj, (val, key) =>
+    isPlainObject(val) ? mapValuesDeep(val, iterator) : iterator(val, key!, obj)
+  );
+
+export async function generateSampleDocuments(
+  options: DeepPartial<GenerateOptions> & {
+    collectTelemetryParams: CollectTelemetryParams;
+  }
+) {
+  const { collectTelemetryParams, ...preferredOptions } = options;
+
+  const opts: GenerateOptions = defaultsDeep(
+    {
+      days: 100,
+      instances: 50,
+      variation: {
+        min: 0.1,
+        max: 4
+      }
+    },
+    preferredOptions
+  );
+
+  const sample = await collectDataTelemetry(collectTelemetryParams);
+
+  const dateOfScriptExecution = new Date();
+
+  return flatten(
+    range(0, opts.instances).map(instanceNo => {
+      const instanceId = uuid.v4();
+      const defaults = {
+        cluster_uuid: instanceId,
+        stack_stats: {
+          kibana: {
+            versions: {
+              version: '8.0.0'
+            }
+          }
+        }
+      };
+
+      const instanceVariation =
+        Math.random() * (opts.variation.max - opts.variation.min) +
+        opts.variation.min;
+
+      return range(0, opts.days).map(dayNo => {
+        const dailyGrowth = Math.pow(1.005, opts.days - 1 - dayNo);
+
+        const timestamp = Date.UTC(
+          dateOfScriptExecution.getFullYear(),
+          dateOfScriptExecution.getMonth(),
+          -dayNo
+        );
+
+        const generated = mapValuesDeep(omit(sample, 'versions'), value =>
+          randomize(value, instanceVariation, dailyGrowth)
+        );
+
+        return merge({}, defaults, {
+          timestamp,
+          stack_stats: {
+            kibana: {
+              plugins: {
+                apm: merge({}, sample, generated)
+              }
+            }
+          }
+        });
+      });
+    })
+  );
+}