[8.0] Add API tests for dependencies metadata (#116648) | Add API tes…

…ts for top dependencies (#116788) | [APM] Adding api tests for error group (#116771) | [APM] Synthtrace ES Client (#116770) | Revert "[APM] Adding api tests for error grou (#117257)

* Add API tests for dependencies metadata (#116648)

* Add API tests for dependencies metadata

* Rename metadata tests to event metadata

Co-authored-by: Nathan L Smith <nathan.smith@elastic.co>

* Add API tests for top dependencies (#116788)

* Add top dependencies API tests

Co-authored-by: Søren Louv-Jansen <sorenlouv@gmail.com>

* [APM] Adding api tests for error group (#116771)

* adding api tests for error group

* addresing pr changes

* [APM] Synthtrace ES Client (#116770)

Co-authored-by: Søren Louv-Jansen <sorenlouv@gmail.com>

* Revert "[APM] Adding api tests for error group (#116771)"

This reverts commit f2402ce.

* [APM] Apm errors api tests (#116764)

* changes after review

* move file to errors folder

* [APM] Fixing synthtrace import location (#117017)

Co-authored-by: Giorgos Bamparopoulos <georgios.bamparopoulos@elastic.co>
Co-authored-by: Nathan L Smith <nathan.smith@elastic.co>
Co-authored-by: Cauê Marcondes <55978943+cauemarcondes@users.noreply.github.com>
Co-authored-by: Dario Gieselaar <dario.gieselaar@elastic.co>
Co-authored-by: Tyler Smalley <tyler.smalley@elastic.co>
Co-authored-by: Miriam <31922082+MiriamAparicio@users.noreply.github.com>

packages/elastic-apm-synthtrace/src/index.ts

@@ -13,3 +13,7 @@ export { getSpanDestinationMetrics } from './lib/utils/get_span_destination_metr
 export { getObserverDefaults } from './lib/defaults/get_observer_defaults';
 export { toElasticsearchOutput } from './lib/output/to_elasticsearch_output';
 export { getBreakdownMetrics } from './lib/utils/get_breakdown_metrics';
+export { cleanWriteTargets } from './lib/utils/clean_write_targets';
+export { getWriteTargets } from './lib/utils/get_write_targets';
+export { SynthtraceEsClient } from './lib/client/synthtrace_es_client';
+export { createLogger, LogLevel } from './lib/utils/create_logger';

packages/elastic-apm-synthtrace/src/lib/client/synthtrace_es_client.ts

@@ -0,0 +1,55 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { Client } from '@elastic/elasticsearch';
+import { uploadEvents } from '../../scripts/utils/upload_events';
+import { Fields } from '../entity';
+import { cleanWriteTargets } from '../utils/clean_write_targets';
+import { getBreakdownMetrics } from '../utils/get_breakdown_metrics';
+import { getSpanDestinationMetrics } from '../utils/get_span_destination_metrics';
+import { getTransactionMetrics } from '../utils/get_transaction_metrics';
+import { getWriteTargets } from '../utils/get_write_targets';
+import { Logger } from '../utils/logger';
+
+export class SynthtraceEsClient {
+  constructor(private readonly client: Client, private readonly logger: Logger) {}
+
+  private getWriteTargets() {
+    return getWriteTargets({ client: this.client });
+  }
+
+  clean() {
+    return this.getWriteTargets().then((writeTargets) =>
+      cleanWriteTargets({ client: this.client, writeTargets, logger: this.logger })
+    );
+  }
+
+  async index(events: Fields[]) {
+    const eventsToIndex = [
+      ...events,
+      ...getTransactionMetrics(events),
+      ...getSpanDestinationMetrics(events),
+      ...getBreakdownMetrics(events),
+    ];
+
+    const writeTargets = await this.getWriteTargets();
+
+    await uploadEvents({
+      batchSize: 1000,
+      client: this.client,
+      clientWorkers: 5,
+      events: eventsToIndex,
+      logger: this.logger,
+      writeTargets,
+    });
+
+    return this.client.indices.refresh({
+      index: Object.values(writeTargets),
+    });
+  }
+}

packages/elastic-apm-synthtrace/src/lib/utils/logger.ts

@@ -0,0 +1,67 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { isPromise } from 'util/types';
+
+export enum LogLevel {
+  trace = 0,
+  debug = 1,
+  info = 2,
+  error = 3,
+}
+
+function getTimeString() {
+  return `[${new Date().toLocaleTimeString()}]`;
+}
+
+export function createLogger(logLevel: LogLevel) {
+  function logPerf(name: string, start: bigint) {
+    // eslint-disable-next-line no-console
+    console.debug(
+      getTimeString(),
+      `${name}: ${Number(process.hrtime.bigint() - start) / 1000000}ms`
+    );
+  }
+  return {
+    perf: <T extends any>(name: string, cb: () => T): T => {
+      if (logLevel <= LogLevel.trace) {
+        const start = process.hrtime.bigint();
+        const val = cb();
+        if (isPromise(val)) {
+          val.then(() => {
+            logPerf(name, start);
+          });
+        } else {
+          logPerf(name, start);
+        }
+        return val;
+      }
+      return cb();
+    },
+    debug: (...args: any[]) => {
+      if (logLevel <= LogLevel.debug) {
+        // eslint-disable-next-line no-console
+        console.debug(getTimeString(), ...args);
+      }
+    },
+    info: (...args: any[]) => {
+      if (logLevel <= LogLevel.info) {
+        // eslint-disable-next-line no-console
+        console.log(getTimeString(), ...args);
+      }
+    },
+    error: (...args: any[]) => {
+      if (logLevel <= LogLevel.error) {
+        // eslint-disable-next-line no-console
+        console.log(getTimeString(), ...args);
+      }
+    },
+  };
+}
+
+export type Logger = ReturnType<typeof createLogger>;

packages/elastic-apm-synthtrace/src/scripts/run.ts

@@ -7,7 +7,7 @@
  */
 import datemath from '@elastic/datemath';
 import yargs from 'yargs/yargs';
-import { cleanWriteTargets } from './utils/clean_write_targets';
+import { cleanWriteTargets } from '../lib/utils/clean_write_targets';
 import { intervalToMs } from './utils/interval_to_ms';
 import { getCommonResources } from './utils/get_common_resources';
 import { startHistoricalDataUpload } from './utils/start_historical_data_upload';

packages/elastic-apm-synthtrace/src/scripts/utils/get_common_resources.ts

@@ -8,9 +8,9 @@
 
 import { Client } from '@elastic/elasticsearch';
 import { getScenario } from './get_scenario';
-import { getWriteTargets } from './get_write_targets';
+import { getWriteTargets } from '../../lib/utils/get_write_targets';
 import { intervalToMs } from './interval_to_ms';
-import { createLogger, LogLevel } from './logger';
+import { createLogger, LogLevel } from '../../lib/utils/create_logger';
 
 export async function getCommonResources({
   file,

packages/elastic-apm-synthtrace/src/scripts/utils/get_scenario.ts

@@ -7,7 +7,7 @@
  */
 import Path from 'path';
 import { Fields } from '../../lib/entity';
-import { Logger } from './logger';
+import { Logger } from '../../lib/utils/create_logger';
 
 export type Scenario = (options: { from: number; to: number }) => Fields[];
 

packages/elastic-apm-synthtrace/src/scripts/utils/start_historical_data_upload.ts

@@ -10,7 +10,7 @@ import pLimit from 'p-limit';
 import Path from 'path';
 import { Worker } from 'worker_threads';
 import { ElasticsearchOutputWriteTargets } from '../../lib/output/to_elasticsearch_output';
-import { Logger, LogLevel } from './logger';
+import { Logger, LogLevel } from '../../lib/utils/create_logger';
 
 export async function startHistoricalDataUpload({
   from,

packages/elastic-apm-synthtrace/src/scripts/utils/start_live_data_upload.ts

@@ -11,7 +11,7 @@ import { partition } from 'lodash';
 import { Fields } from '../../lib/entity';
 import { ElasticsearchOutputWriteTargets } from '../../lib/output/to_elasticsearch_output';
 import { Scenario } from './get_scenario';
-import { Logger } from './logger';
+import { Logger } from '../../lib/utils/create_logger';
 import { uploadEvents } from './upload_events';
 
 export function startLiveDataUpload({

packages/elastic-apm-synthtrace/src/scripts/utils/upload_events.ts

@@ -14,7 +14,7 @@ import {
   ElasticsearchOutputWriteTargets,
   toElasticsearchOutput,
 } from '../../lib/output/to_elasticsearch_output';
-import { Logger } from './logger';
+import { Logger } from '../../lib/utils/create_logger';
 
 export function uploadEvents({
   events,
@@ -56,23 +56,17 @@ export function uploadEvents({
         );
       })
     )
-  )
-    .then((results) => {
-      const errors = results
-        .flatMap((result) => result.items)
-        .filter((item) => !!item.index?.error)
-        .map((item) => item.index?.error);
+  ).then((results) => {
+    const errors = results
+      .flatMap((result) => result.items)
+      .filter((item) => !!item.index?.error)
+      .map((item) => item.index?.error);
 
-      if (errors.length) {
-        logger.error(inspect(errors.slice(0, 10), { depth: null }));
-        throw new Error('Failed to upload some items');
-      }
+    if (errors.length) {
+      logger.error(inspect(errors.slice(0, 10), { depth: null }));
+      throw new Error('Failed to upload some items');
+    }
 
-      logger.debug(`Uploaded ${events.length} in ${new Date().getTime() - time}ms`);
-    })
-    .catch((err) => {
-      // eslint-disable-next-line no-console
-      console.error(err);
-      process.exit(1);
-    });
+    logger.debug(`Uploaded ${events.length} in ${new Date().getTime() - time}ms`);
+  });
 }

packages/elastic-apm-synthtrace/src/scripts/utils/upload_next_batch.ts

@@ -11,7 +11,7 @@ import { Client } from '@elastic/elasticsearch';
 import { workerData } from 'worker_threads';
 import { ElasticsearchOutputWriteTargets } from '../../lib/output/to_elasticsearch_output';
 import { getScenario } from './get_scenario';
-import { createLogger, LogLevel } from './logger';
+import { createLogger, LogLevel } from '../../lib/utils/create_logger';
 import { uploadEvents } from './upload_events';
 
 const { bucketFrom, bucketTo, file, logLevel, target, writeTargets, clientWorkers, batchSize } =