From e22f369dc33121d1b71ceaebdeb532599c8e6c6a Mon Sep 17 00:00:00 2001 From: Ines Fazlic Date: Mon, 4 Mar 2024 13:23:07 +0000 Subject: [PATCH] feat(fargate): use adot image for otel (#2481) --- package-lock.json | 60 +--- .../artillery-plugin-publish-metrics/index.js | 9 +- .../lib/open-telemetry/index.js | 8 +- .../lib/open-telemetry/tracing/base.js | 2 - .../open-telemetry/translators/vendor-adot.js | 150 +++++++++ .../vendor-otel.js} | 6 +- .../test/unit/adot-translators.js | 315 ++++++++++++++++++ .../lib/platform/aws-ecs/legacy/aws-util.js | 28 +- .../lib/platform/aws-ecs/legacy/bom.js | 19 +- .../platform/aws-ecs/legacy/create-test.js | 5 +- .../platform/aws-ecs/legacy/run-cluster.js | 166 +++++---- packages/artillery/lib/util.js | 12 +- packages/artillery/package.json | 3 +- .../test/cloud-e2e/fargate/adot.test.js | 122 +++++++ .../fargate/fixtures/adot/adot-dd-pass.yml | 22 ++ .../fargate/fixtures/adot/helpers.js | 67 ++++ .../artillery/test/unit/fargate-bom.test.js | 149 +++++++++ 17 files changed, 1006 insertions(+), 137 deletions(-) create mode 100644 packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-adot.js rename packages/artillery-plugin-publish-metrics/lib/open-telemetry/{vendor-translators.js => translators/vendor-otel.js} (95%) create mode 100644 packages/artillery-plugin-publish-metrics/test/unit/adot-translators.js create mode 100644 packages/artillery/test/cloud-e2e/fargate/adot.test.js create mode 100644 packages/artillery/test/cloud-e2e/fargate/fixtures/adot/adot-dd-pass.yml create mode 100644 packages/artillery/test/cloud-e2e/fargate/fixtures/adot/helpers.js create mode 100644 packages/artillery/test/unit/fargate-bom.test.js diff --git a/package-lock.json b/package-lock.json index b1fdf7674b..e37b7bc6ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5724,35 +5724,6 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, - "node_modules/acorn-node": { - "version": "1.8.2", - "resolved": "https://registry.npmjs.org/acorn-node/-/acorn-node-1.8.2.tgz", - "integrity": "sha512-8mt+fslDufLYntIoPAaIMUe/lrbrehIiwmR3t2k9LljIzoigEPF27eLk2hy8zSGzmR/ogr7zbRKINMo1u0yh5A==", - "dependencies": { - "acorn": "^7.0.0", - "acorn-walk": "^7.0.0", - "xtend": "^4.0.2" - } - }, - "node_modules/acorn-node/node_modules/acorn": { - "version": "7.4.1", - "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", - "integrity": "sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A==", - "bin": { - "acorn": "bin/acorn" - }, - "engines": { - "node": ">=0.4.0" - } - }, - "node_modules/acorn-node/node_modules/acorn-walk": { - "version": "7.2.0", - "resolved": "https://registry.npmjs.org/acorn-walk/-/acorn-walk-7.2.0.tgz", - "integrity": "sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA==", - "engines": { - "node": ">=0.4.0" - } - }, "node_modules/acorn-walk": { "version": "8.2.0", "license": "MIT", @@ -8163,14 +8134,6 @@ "node": ">=10" } }, - "node_modules/defined": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/defined/-/defined-1.0.1.tgz", - "integrity": "sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q==", - "funding": { - "url": "https://github.com/sponsors/ljharb" - } - }, "node_modules/delayed-stream": { "version": "1.0.0", "license": "MIT", @@ -8250,22 +8213,6 @@ "npm": "1.2.8000 || >= 1.4.16" } }, - "node_modules/detective": { - "version": "5.1.0", - "resolved": "https://registry.npmjs.org/detective/-/detective-5.1.0.tgz", - "integrity": "sha512-TFHMqfOvxlgrfVzTEkNBSh9SvSNX/HfF4OFI2QFGCyPm02EsyILqnUeb5P6q7JZ3SFNTBL5t2sePRgrN4epUWQ==", - "dependencies": { - "acorn-node": "^1.3.0", - "defined": "^1.0.0", - "minimist": "^1.1.1" - }, - "bin": { - "detective": "bin/detective.js" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/detective-amd": { "version": "5.0.2", "resolved": "https://registry.npmjs.org/detective-amd/-/detective-amd-5.0.2.tgz", @@ -10817,7 +10764,8 @@ }, "node_modules/got": { "version": "11.8.5", - "license": "MIT", + "resolved": "https://registry.npmjs.org/got/-/got-11.8.5.tgz", + "integrity": "sha512-o0Je4NvQObAuZPHLFoRSkdG2lTgtcynqymzg2Vupdx6PorhaT5MCbIyXG6d4D94kk8ZG57QeosgdiqfJWhEhlQ==", "dependencies": { "@sindresorhus/is": "^4.0.0", "@szmarczak/http-timer": "^4.0.5", @@ -20705,6 +20653,7 @@ }, "node_modules/xtend": { "version": "4.0.2", + "dev": true, "license": "MIT", "engines": { "node": ">=0.4" @@ -21598,7 +21547,7 @@ "csv-parse": "^4.16.3", "debug": "^4.3.1", "dependency-tree": "^10.0.9", - "detective": "^5.1.0", + "detective-es6": "^4.0.1", "dotenv": "^16.0.1", "esbuild-wasm": "^0.19.8", "eventemitter3": "^4.0.4", @@ -21629,6 +21578,7 @@ "eslint-plugin-prettier": "^4.0.0", "execa": "^0.10.0", "get-bin-path": "^5.1.0", + "got": "^11.8.5", "rewiremock": "^3.14.3", "sinon": "^4.5.0", "tap": "^16.3.7", diff --git a/packages/artillery-plugin-publish-metrics/index.js b/packages/artillery-plugin-publish-metrics/index.js index f4bd7c429d..68ba680095 100644 --- a/packages/artillery-plugin-publish-metrics/index.js +++ b/packages/artillery-plugin-publish-metrics/index.js @@ -6,6 +6,11 @@ const NS = 'plugin:publish-metrics'; const debug = require('debug')(NS); const A = require('async'); +const { + getADOTRelevantReporterConfigs, + resolveADOTConfigSettings +} = require('./lib/open-telemetry/translators/vendor-adot'); + // List of reporters that use OpenTelemetry const REPORTERS_USING_OTEL = [ 'open-telemetry', @@ -16,7 +21,9 @@ const REPORTERS_USING_OTEL = [ ]; module.exports = { Plugin, - LEGACY_METRICS_FORMAT: false + LEGACY_METRICS_FORMAT: false, + getADOTRelevantReporterConfigs, + resolveADOTConfigSettings }; function Plugin(script, events) { diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js index 529bc15618..c55cef58f6 100644 --- a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js @@ -1,6 +1,6 @@ 'use strict'; -const vendorTranslators = require('./vendor-translators'); +const { vendorTranslators } = require('./translators/vendor-otel'); const { diag, DiagConsoleLogger, @@ -71,6 +71,7 @@ class OTelReporter { if (this.tracesConfig) { global.artillery.OTEL_TRACING_ENABLED = true; } + // Warn if traces are configured in multiple reporters this.warnIfDuplicateTracesConfigured(this.translatedConfigsList); @@ -156,6 +157,11 @@ class OTelReporter { if (!this.metricsConfig && !this.tracesConfig) { return done(); } + + // Waiting for flush period to complete here rather than in trace/metric reporters + this.debug('Waiting for flush period to end'); + await new Promise((resolve) => setTimeout(resolve, 10000)); + if (this.metricReporter) { await this.metricReporter.cleanup(); } diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js index 5b4e83a1ce..a61cc781ac 100644 --- a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js @@ -178,8 +178,6 @@ class OTelTraceBase { } this.debug('Pending traces done'); - this.debug('Waiting for flush period to complete'); - await sleep(5000); } } diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-adot.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-adot.js new file mode 100644 index 0000000000..35bd4572fb --- /dev/null +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-adot.js @@ -0,0 +1,150 @@ +'use strict'; + +const ADOTSupportedTraceReporters = ['datadog']; +const ADOTSupportedMetricReporters = []; + +// Getting the relevant reporter configurations from full publish-metrics configuration + +function getADOTRelevantReporterConfigs(publishMetricsConfig) { + const configs = publishMetricsConfig.filter( + (reporterConfig) => + (ADOTSupportedTraceReporters.includes(reporterConfig.type) && + reporterConfig.traces) || + (ADOTSupportedMetricReporters.includes(reporterConfig.type) && + reporterConfig.metrics) + ); + + return configs; +} + +// Resolve the configuration settings for ADOT + +function resolveADOTConfigSettings(options) { + try { + const adotConfig = getADOTConfig(options.configList); // options.configList ( array of those reporter configurations from publish-metrics config that require ADOT ) + const adotEnvVars = getADOTEnvVars(options.configList, options.dotenv); // options.dotenv (object with environment variables from user provided dotenv file) + return { adotConfig, adotEnvVars }; + } catch (err) { + throw new Error(err); + } +} + +// Assembling the configuration for ADOT (in OTel Collector format) + +function getADOTConfig(adotRelevantConfigs) { + const translatedVendorConfigs = adotRelevantConfigs.map((config) => + vendorToCollectorConfigTranslators[config.type](config) + ); + + // Different vendors can be used for metrics and tracing so we need to merge configs from each vendor into one collector config + const finalADOTConfig = JSON.parse(JSON.stringify(collectorConfigTemplate)); + + translatedVendorConfigs.forEach((config) => { + finalADOTConfig.processors = Object.assign( + finalADOTConfig.processors, + config.processors + ); + finalADOTConfig.exporters = Object.assign( + finalADOTConfig.exporters, + config.exporters + ); + finalADOTConfig.service.pipelines = Object.assign( + finalADOTConfig.service.pipelines, + config.service.pipelines + ); + }); + return finalADOTConfig; +} + +const collectorConfigTemplate = { + receivers: { + otlp: { + protocols: { + http: { + endpoint: '0.0.0.0:4318' + }, + grpc: { + endpoint: '0.0.0.0:4317' + } + } + } + }, + processors: {}, + exporters: {}, + service: { + pipelines: {} + } +}; + +// Map of functions that translate vendor-specific configuration to OpenTelemetry Collector configuration to be used by ADOT +const vendorToCollectorConfigTranslators = { + datadog: (config) => { + const collectorConfig = JSON.parse(JSON.stringify(collectorConfigTemplate)); + if (config.traces) { + collectorConfig.processors['batch/trace'] = { + timeout: '10s', + send_batch_max_size: 1024, + send_batch_size: 200 + }; + collectorConfig.exporters['datadog/api'] = { + traces: { + trace_buffer: 100 + }, + api: { + key: '${env:DD_API_KEY}' + } + }; + collectorConfig.service.pipelines.traces = { + receivers: ['otlp'], + processors: ['batch/trace'], + exporters: ['datadog/api'] + }; + } + return collectorConfig; + } +}; + +// Handling vendor specific environment variables needed for ADOT configuration (e.g. Authentication keys/tokens that can be provided in the script ) + +function getADOTEnvVars(adotRelevantconfigs, dotenv) { + const envVars = {}; + try { + adotRelevantconfigs.forEach((config) => { + const vendorVars = vendorSpecificEnvVarsForCollector[config.type]( + config, + dotenv + ); + Object.assign(envVars, vendorVars); + }); + } catch (err) { + // We warn here instead of throwing because in the future we will support providing these variables through secrets + console.warn(err.message); + } + return envVars; +} + +const vendorSpecificEnvVarsForCollector = { + datadog: (config, dotenv) => { + const apiKey = config.apiKey || dotenv?.DD_API_KEY; + // We validate API key here for Datadog (for now) because it is only required if Datadog tracing is set with test running on Fargate. (for local runs user configures their own agent, and for metrics if apiKey is not provided the reporter defaults to sending data to agent) + if (!apiKey) { + throw new Error( + "Datadog reporter Error: Missing Datadog API key. Provide it under 'apiKey' setting in your script or under 'DD_API_KEY' environment variable set in your dotenv file." + ); + } + return { DD_API_KEY: apiKey }; + } +}; + +module.exports = { + getADOTRelevantReporterConfigs, + resolveADOTConfigSettings, + // All func and vars below exported for testing purposes + getADOTEnvVars, + vendorSpecificEnvVarsForCollector, + getADOTConfig, + vendorToCollectorConfigTranslators, + ADOTSupportedTraceReporters, + ADOTSupportedMetricReporters, + collectorConfigTemplate +}; diff --git a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/vendor-translators.js b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-otel.js similarity index 95% rename from packages/artillery-plugin-publish-metrics/lib/open-telemetry/vendor-translators.js rename to packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-otel.js index ad71e65f7c..5d3b951cd3 100644 --- a/packages/artillery-plugin-publish-metrics/lib/open-telemetry/vendor-translators.js +++ b/packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-otel.js @@ -1,6 +1,6 @@ 'use strict'; -// Map of functions that translate vendor-specific configuration to OpenTelemetry configuration +// Map of functions that translate vendor-specific reporter configuration to OpenTelemetry reporter configuration const vendorTranslators = { honeycomb: (config) => { if (config.enabled === false) { @@ -115,4 +115,6 @@ function attributeListToObject(attributeList, reporterType) { return attributes; } -module.exports = vendorTranslators; +module.exports = { + vendorTranslators +}; diff --git a/packages/artillery-plugin-publish-metrics/test/unit/adot-translators.js b/packages/artillery-plugin-publish-metrics/test/unit/adot-translators.js new file mode 100644 index 0000000000..4d771b71eb --- /dev/null +++ b/packages/artillery-plugin-publish-metrics/test/unit/adot-translators.js @@ -0,0 +1,315 @@ +const { test } = require('tap'); + +const { + getADOTRelevantReporterConfigs, + vendorSpecificEnvVarsForCollector, + getADOTEnvVars, + vendorToCollectorConfigTranslators, + getADOTConfig, + resolveADOTConfigSettings, + collectorConfigTemplate +} = require('../../lib/open-telemetry/translators/vendor-adot'); + +// Test getADOTRelevantReporterConfigs +test('getADOTRelevantReporterConfigs correctly filters out unsupported reporter configurations from a list of reporter configs', async (t) => { + const pmConfigList = [ + { + type: 'datadog', + traces: {} + }, + { + type: 'unsupportedVendor', + traces: {} + }, + { + type: 'datadog' + } + ]; + const result = getADOTRelevantReporterConfigs(pmConfigList); + t.same(result, [ + { + type: 'datadog', + traces: {} + } + ]); +}); + +test('getADOTRelevantReporterConfigs returns empty list when no relevant reporter configurations are found', async (t) => { + const pmConfigList = [ + { + type: 'unsupportedVendor', + traces: {} + } + ]; + const result = getADOTRelevantReporterConfigs(pmConfigList); + t.same(result, []); +}); + +test('getADOTRelevantReporterConfigs does not return configuration when the supported signal type is not configured', async (t) => { + const pmConfigList = [ + { + type: 'datadog' + } + ]; + const result = getADOTRelevantReporterConfigs(pmConfigList); + t.same(result, []); +}); + +// Test vendorSpecificEnvVarsForCollector +test('when vendorSpecificEnvVarsForCollector is called with needed parameters, it returns an object with the DD_API_KEY', async (t) => { + const config = { + type: 'datadog', + tracing: {} + }; + const dotenv = { + DD_API_KEY: '123' + }; + const result = vendorSpecificEnvVarsForCollector.datadog(config, dotenv); + t.same(result, { DD_API_KEY: '123' }); +}); + +test('when vendorSpecificEnvVarsForCollector is called without apiKey or DD_API_KEY, it throws an error', async (t) => { + const config = { + type: 'datadog', + tracing: {} + }; + const dotenv = {}; + t.throws(() => vendorSpecificEnvVarsForCollector.datadog(config, dotenv), { + message: + "Datadog reporter Error: Missing Datadog API key. Provide it under 'apiKey' setting in your script or under 'DD_API_KEY' environment variable set in your dotenv file." + }); +}); + +test('when vendorSpecificEnvVarsForCollector is called with an apiKey but and an empty dotenv object, it returns an object with the DD_API_KEY', async (t) => { + const config = { + type: 'datadog', + apiKey: '123', + tracing: {} + }; + const dotenv = {}; + const result = vendorSpecificEnvVarsForCollector.datadog(config, dotenv); + t.same(result, { DD_API_KEY: '123' }); +}); + +test('when vendorSpecificEnvVarsForCollector is called with an apiKey and DD_API_KEY, it returns an object with the DD_API_KEY as the apiKey', async (t) => { + const config = { + type: 'datadog', + apiKey: '123', + tracing: {} + }; + const dotenv = { + DD_API_KEY: '456' + }; + const result = vendorSpecificEnvVarsForCollector.datadog(config, dotenv); + t.same(result, { DD_API_KEY: '123' }); +}); + +// Test getADOTEnvVars +test('when getADOTEnvVars is called with a list of adotRelevantconfigs and a dotenv, it returns an object with the environment variables for the relevant vendors', async (t) => { + const adotRelevantconfigs = [ + { + type: 'datadog', + tracing: {} + } + ]; + const dotenv = { + DD_API_KEY: '123' + }; + const result = getADOTEnvVars(adotRelevantconfigs, dotenv); + t.same(result, { DD_API_KEY: '123' }); +}); + +// Maybe remove throwing an error in the function itself as it propagates to the resolveADOTConfigSettings and then we can handle it there +test('if an error happens in getADOTEnvVars it logs the error message and returns the current state of envVars variable ', async (t) => { + const adotRelevantconfigs = [ + { + type: 'datadog', + tracing: {} + } + ]; + const dotenv = {}; + const result = getADOTEnvVars(adotRelevantconfigs, dotenv); + t.same(result, {}); +}); + +test('when getADOTEnvVars is called with an empty list of adotRelevantconfigs and an empty dotenv object it returns an empty object', async (t) => { + const adotRelevantconfigs = []; + const dotenv = {}; + const result = getADOTEnvVars(adotRelevantconfigs, dotenv); + t.same(result, {}); +}); + +// Test vendorToCollectorConfigTranslators +test('when vendorToCollectorConfigTranslators is called with a datadog config, it returns an object with the correct properties', async (t) => { + const config = { + type: 'datadog', + traces: {} + }; + const result = vendorToCollectorConfigTranslators.datadog(config); + t.same(result, { + receivers: { + otlp: { + protocols: { + http: { + endpoint: '0.0.0.0:4318' + }, + grpc: { + endpoint: '0.0.0.0:4317' + } + } + } + }, + processors: { + 'batch/trace': { + timeout: '10s', + send_batch_max_size: 1024, + send_batch_size: 200 + } + }, + exporters: { + 'datadog/api': { + traces: { + trace_buffer: 100 + }, + api: { + key: '${env:DD_API_KEY}' + } + } + }, + service: { + pipelines: { + traces: { + receivers: ['otlp'], + processors: ['batch/trace'], + exporters: ['datadog/api'] + } + } + } + }); +}); + +test('when vendorToCollectorConfigTranslators is called with a datadog config that has no traces configured it returns the collector config template', async (t) => { + const config = { + type: 'datadog' + }; + const result = vendorToCollectorConfigTranslators.datadog(config); + t.same(result, collectorConfigTemplate); +}); + +// Test getADOTConfig +test('when getADOTConfig is called with a list of adotRelevantConfigs, it returns an object with the correct properties', async (t) => { + const adotRelevantConfigs = [ + { + type: 'datadog', + traces: {} + } + ]; + const result = getADOTConfig(adotRelevantConfigs); + t.same(result, { + receivers: { + otlp: { + protocols: { + http: { + endpoint: '0.0.0.0:4318' + }, + grpc: { + endpoint: '0.0.0.0:4317' + } + } + } + }, + processors: { + 'batch/trace': { + timeout: '10s', + send_batch_max_size: 1024, + send_batch_size: 200 + } + }, + exporters: { + 'datadog/api': { + traces: { + trace_buffer: 100 + }, + api: { + key: '${env:DD_API_KEY}' + } + } + }, + service: { + pipelines: { + traces: { + receivers: ['otlp'], + processors: ['batch/trace'], + exporters: ['datadog/api'] + } + } + } + }); +}); + +test('when getADOTConfig is called with an empty list of adotRelevantConfigs, it returns an object that is same as collector config template', async (t) => { + const adotRelevantConfigs = []; + const result = getADOTConfig(adotRelevantConfigs); + t.same(result, collectorConfigTemplate); +}); + +// Test resolveADOTConfigSettings +test('when resolveADOTConfigSettings is called with a configList and a dotenv object, it returns an object with the correct adotConfig and adotEnvVars', async (t) => { + const options = { + configList: [ + { + type: 'datadog', + traces: {} + } + ], + dotenv: { + DD_API_KEY: '123' + } + }; + const result = resolveADOTConfigSettings(options); + t.same(result, { + adotConfig: { + receivers: { + otlp: { + protocols: { + http: { + endpoint: '0.0.0.0:4318' + }, + grpc: { + endpoint: '0.0.0.0:4317' + } + } + } + }, + processors: { + 'batch/trace': { + timeout: '10s', + send_batch_max_size: 1024, + send_batch_size: 200 + } + }, + exporters: { + 'datadog/api': { + traces: { + trace_buffer: 100 + }, + api: { + key: '${env:DD_API_KEY}' + } + } + }, + service: { + pipelines: { + traces: { + receivers: ['otlp'], + processors: ['batch/trace'], + exporters: ['datadog/api'] + } + } + } + }, + adotEnvVars: { + DD_API_KEY: '123' + } + }); +}); diff --git a/packages/artillery/lib/platform/aws-ecs/legacy/aws-util.js b/packages/artillery/lib/platform/aws-ecs/legacy/aws-util.js index 8c55daf733..520de3023f 100644 --- a/packages/artillery/lib/platform/aws-ecs/legacy/aws-util.js +++ b/packages/artillery/lib/platform/aws-ecs/legacy/aws-util.js @@ -9,7 +9,8 @@ module.exports = { ensureParameterExists, parameterExists, putParameter, - getParameter + getParameter, + deleteParameter }; // Wraps ecs.describeTasks to support more than 100 task ARNs in params.tasks @@ -124,3 +125,28 @@ async function getParameter(path, region) { } } } + +async function deleteParameter(path, region) { + if (region) { + AWS.config.update({ region }); + } + + const ssm = new AWS.SSM({ apiVersion: '2014-11-06' }); + + try { + const ssmResponse = await ssm + .deleteParameter({ + Name: path + }) + .promise(); + + debug({ ssmResponse }); + return ssmResponse; + } catch (ssmErr) { + if (ssmErr.code === 'ParameterNotFound') { + return false; + } else { + throw ssmErr; + } + } +} \ No newline at end of file diff --git a/packages/artillery/lib/platform/aws-ecs/legacy/bom.js b/packages/artillery/lib/platform/aws-ecs/legacy/bom.js index f60b2b0326..e4c5fba48e 100644 --- a/packages/artillery/lib/platform/aws-ecs/legacy/bom.js +++ b/packages/artillery/lib/platform/aws-ecs/legacy/bom.js @@ -14,6 +14,7 @@ const BUILTIN_ENGINES = require('./plugins').getOfficialEngines(); const Table = require('cli-table3'); +const { resolveConfigTemplates } = require('../../../../util'); // NOTE: Code below presumes that all paths are absolute //Tests in Fargate run on ubuntu, which uses posix paths @@ -32,12 +33,14 @@ function createBOM(absoluteScriptPath, extraFiles, opts, callback) { return next(null, { opts: { scriptData, - absoluteScriptPath + absoluteScriptPath, + flags: opts.flags }, localFilePaths: [absoluteScriptPath], npmModules: [] }); }, + applyScriptChanges, getPlugins, getCustomEngines, getCustomJsDependencies, @@ -137,7 +140,8 @@ function createBOM(absoluteScriptPath, extraFiles, opts, callback) { return callback(null, { files: _.uniqWith(files, _.isEqual), modules: _.uniq(context.npmModules), - pkgDeps: context.pkgDeps + pkgDeps: context.pkgDeps, + fullyResolvedConfig: context.opts.scriptData.config }); } ); @@ -148,6 +152,15 @@ function isLocalModule(modName) { return modName.startsWith('.'); } +function applyScriptChanges(context, next) { + resolveConfigTemplates(context.opts.scriptData, context.opts.flags).then( + (resolvedConfig) => { + context.opts.scriptData = resolvedConfig; + return next(null, context); + } + ); +} + function getPlugins(context, next) { let environmentPlugins = _.reduce( _.get(context, 'opts.scriptData.config.environments', {}), @@ -459,4 +472,4 @@ function prettyPrint(manifest) { artillery.log(); } -module.exports = { createBOM, commonPrefix, prettyPrint }; +module.exports = { createBOM, commonPrefix, prettyPrint, applyScriptChanges }; diff --git a/packages/artillery/lib/platform/aws-ecs/legacy/create-test.js b/packages/artillery/lib/platform/aws-ecs/legacy/create-test.js index 23d2eef7f7..3b8a60b477 100644 --- a/packages/artillery/lib/platform/aws-ecs/legacy/create-test.js +++ b/packages/artillery/lib/platform/aws-ecs/legacy/create-test.js @@ -37,7 +37,8 @@ async function createTest(scriptPath, options, callback) { originalScriptPath: scriptPath, name: options.name, // test name, eg simple-bom or aht_$UUID manifestPath: options.manifestPath, - packageJsonPath: options.packageJsonPath + packageJsonPath: options.packageJsonPath, + flags: options.flags }; if (typeof options.config === 'string') { @@ -82,7 +83,7 @@ function prepareManifest(context, callback) { createBOM( fileToAnalyse, extraFiles, - { packageJsonPath: context.packageJsonPath }, + { packageJsonPath: context.packageJsonPath, flags: context.flags}, (err, bom) => { debug(err); debug(bom); diff --git a/packages/artillery/lib/platform/aws-ecs/legacy/run-cluster.js b/packages/artillery/lib/platform/aws-ecs/legacy/run-cluster.js index 089d118140..99ff4d002c 100644 --- a/packages/artillery/lib/platform/aws-ecs/legacy/run-cluster.js +++ b/packages/artillery/lib/platform/aws-ecs/legacy/run-cluster.js @@ -15,6 +15,11 @@ const moment = require('moment'); const EnsurePlugin = require('artillery-plugin-ensure'); +const { + getADOTRelevantReporterConfigs, + resolveADOTConfigSettings +} = require('artillery-plugin-publish-metrics'); + const EventEmitter = require('events'); const _ = require('lodash'); @@ -36,7 +41,7 @@ const { getBucketName } = require('./util'); const getAccountId = require('../../aws/aws-get-account-id'); const { setCloudwatchRetention } = require('../../aws/aws-cloudwatch'); -const dotenvParse = require('dotenv').parse; +const dotenv = require('dotenv'); const util = require('./util'); @@ -225,10 +230,11 @@ async function tryRunCluster(scriptPath, options, artilleryReporter) { } if (options.dotenv) { - const contents = fs.readFileSync( - path.resolve(process.cwd(), options.dotenv) - ); - context.dotenv = dotenvParse(contents); + const dotEnvPath = path.resolve(process.cwd(), options.dotenv); + dotenv.config({ path: dotEnvPath }); + + const contents = fs.readFileSync(dotEnvPath); + context.dotenv = dotenv.parse(contents); } if (options.bundle) { @@ -643,9 +649,10 @@ async function tryRunCluster(scriptPath, options, artilleryReporter) { await createQueue(context); await checkCustomTaskRole(context); - await ensureTaskExists(context); logProgress('Preparing test bundle...'); await createTestBundle(context); + await createADOTDefinitionIfNeeded(context); + await ensureTaskExists(context); await getManifest(context); await generateTaskOverrides(context); @@ -823,6 +830,13 @@ async function cleanupResources(context) { context.sqsReporter.stop(); } + if (context.adot?.SSMParameterPath) { + await awsUtil.deleteParameter( + context.adot.SSMParameterPath, + context.region + ); + } + if (context.taskArns && context.taskArns.length > 0) { for (const taskArn of context.taskArns) { try { @@ -980,12 +994,14 @@ async function createTestBundle(context) { { name: context.testId, config: context.cliOptions.config, - packageJsonPath: context.packageJsonPath + packageJsonPath: context.packageJsonPath, + flags: context.cliOptions }, - function (err, _result) { + function (err, result) { if (err) { return reject(err); } else { + context.fullyResolvedConfig = result.manifest.fullyResolvedConfig; return resolve(context); } } @@ -993,6 +1009,68 @@ async function createTestBundle(context) { }); } +async function createADOTDefinitionIfNeeded(context) { + const publishMetricsConfig = + context.fullyResolvedConfig.plugins?.['publish-metrics']; + if (!publishMetricsConfig) { + debug('No publish-metrics plugin set, skipping ADOT configuration'); + return context; + } + + const adotRelevantConfigs = + getADOTRelevantReporterConfigs(publishMetricsConfig); + if (adotRelevantConfigs.length === 0) { + debug('No ADOT relevant reporter configs set, skipping ADOT configuration'); + return context; + } + + try { + const { adotEnvVars, adotConfig } = resolveADOTConfigSettings({ + configList: adotRelevantConfigs, + dotenv: { ...context.dotenv } + }); + + context.dotenv = Object.assign(context.dotenv || {}, adotEnvVars); + + context.adot = { + SSMParameterPath: `/artilleryio/OTEL_CONFIG_${context.testId}` + }; + + await awsUtil.putParameter( + context.adot.SSMParameterPath, + JSON.stringify(adotConfig), + 'String', + context.region + ); + + context.adot.taskDefinition = { + name: 'adot-collector', + image: 'amazon/aws-otel-collector', + command: [ + '--config=/etc/ecs/container-insights/otel-task-metrics-config.yaml' + ], + secrets: [ + { + name: 'AOT_CONFIG_CONTENT', + valueFrom: `arn:aws:ssm:${context.region}:${context.accountId}:parameter${context.adot.SSMParameterPath}` + } + ], + logConfiguration: { + logDriver: 'awslogs', + options: { + 'awslogs-group': `${context.logGroupName}/${context.clusterName}`, + 'awslogs-region': context.region, + 'awslogs-stream-prefix': `artilleryio/${context.testId}`, + 'awslogs-create-group': 'true' + } + } + }; + } catch (err) { + throw new Error(err); + } + return context; +} + async function ensureTaskExists(context) { return new Promise((resolve, reject) => { const ecs = new AWS.ECS({ @@ -1097,54 +1175,7 @@ async function ensureTaskExists(context) { } } }, - { - name: 'datadog-agent', - image: 'public.ecr.aws/datadog/agent:7', - environment: [ - { - name: 'DD_API_KEY', - value: 'placeholder' - }, - { - name: 'DD_OTLP_CONFIG_TRACES_ENABLED', - value: 'true' - }, - { - name: 'DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT', - value: '0.0.0.0:4318' - }, - { - name: 'DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_GRPC_ENDPOINT', - value: '0.0.0.0:4317' - }, - { - name: 'DD_APM_ENABLED', - value: 'true' - }, - { - name: 'DD_APM_RECEIVER_PORT', - value: '8126' - }, - { - name: 'DD_APM_TRACE_BUFFER', - value: '100' - }, - { - name: 'DD_SITE', - value: 'datadoghq.com' - } - ], - logConfiguration: { - logDriver: 'awslogs', - options: { - 'awslogs-group': `${context.logGroupName}/${context.clusterName}`, - 'awslogs-region': context.region, - 'awslogs-stream-prefix': `artilleryio/${context.testId}`, - 'awslogs-create-group': 'true', - mode: 'non-blocking' - } - } - } + ...([context.adot?.taskDefinition] || []) ], executionRoleArn: context.taskRoleArn }; @@ -1384,6 +1415,12 @@ async function generateTaskOverrides(context) { const s3path = `s3://${context.s3Bucket}/tests/${ context.namedTest ? context.s3Prefix : context.testId }`; + const adotOverride = [ + { + name: 'adot-collector', + environment: [] + } + ]; const overrides = { containerOverrides: [ @@ -1416,10 +1453,7 @@ async function generateTaskOverrides(context) { } ] }, - { - name: 'datadog-agent', - environment: [] - } + ...(context.adot ? adotOverride : []) ], taskRoleArn: context.taskRoleArn }; @@ -1440,8 +1474,10 @@ async function generateTaskOverrides(context) { } overrides.containerOverrides[0].environment = overrides.containerOverrides[0].environment.concat(extraEnv); - overrides.containerOverrides[1].environment = - overrides.containerOverrides[1].environment.concat(extraEnv); + if (overrides.containerOverrides[1]) { + overrides.containerOverrides[1].environment = + overrides.containerOverrides[1].environment.concat(extraEnv); + } } if (context.cliOptions.launchConfig) { @@ -1449,8 +1485,10 @@ async function generateTaskOverrides(context) { if (lc.environment) { overrides.containerOverrides[0].environment = overrides.containerOverrides[0].environment.concat(lc.environment); - overrides.containerOverrides[1].environment = - overrides.containerOverrides[1].environment.concat(lc.environment); + if (overrides.containerOverrides[1]) { + overrides.containerOverrides[1].environment = + overrides.containerOverrides[1].environment.concat(lc.environment); + } } // diff --git a/packages/artillery/lib/util.js b/packages/artillery/lib/util.js index 2d73a0e030..67d2eb9349 100644 --- a/packages/artillery/lib/util.js +++ b/packages/artillery/lib/util.js @@ -79,20 +79,22 @@ function addDefaultPlugins(script) { const finalScript = _.cloneDeep(script); if (!script.config.plugins) { - finalScript.config.plugins = {} + finalScript.config.plugins = {}; } const additionalPluginsAndOptions = { 'metrics-by-endpoint': { suppressOutput: true, stripQueryString: true } - } + }; - for (const [pluginName, pluginOptions] of Object.entries(additionalPluginsAndOptions)) { + for (const [pluginName, pluginOptions] of Object.entries( + additionalPluginsAndOptions + )) { if (!finalScript.config.plugins[pluginName]) { - finalScript.config.plugins[pluginName] = pluginOptions + finalScript.config.plugins[pluginName] = pluginOptions; } } - return finalScript + return finalScript; } async function resolveConfigTemplates(script, flags) { diff --git a/packages/artillery/package.json b/packages/artillery/package.json index 3aaff5bfba..d930a11911 100644 --- a/packages/artillery/package.json +++ b/packages/artillery/package.json @@ -47,7 +47,7 @@ "test:acceptance": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --timeout=420 test/cli/*.test.js && bash test/lib/run.sh && tap --no-coverage --color test/testcases/plugins/*.test.js", "test": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && npm run test:unit && npm run test:acceptance", "test:windows": "set ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' & npm run test:unit && tap --no-coverage --timeout=420 --color test/cli/*.test.js", - "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=3600 test/cloud-e2e/**/*.test.js", + "test:aws": "export ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' && tap --no-coverage --color --timeout=4200 test/cloud-e2e/**/*.test.js", "test:aws:windows": "set ARTILLERY_TELEMETRY_DEFAULTS='{\"source\":\"test-suite\"}' & tap --no-coverage --timeout=420 --color test/cloud-e2e/fargate/*.test.js --grep \"Run simple-bom\"", "lint": "eslint --ext \".js,.ts,.tsx\" .", "lint-fix": "npm run lint -- --fix" @@ -138,6 +138,7 @@ "eslint-plugin-prettier": "^4.0.0", "execa": "^0.10.0", "get-bin-path": "^5.1.0", + "got": "^11.8.5", "rewiremock": "^3.14.3", "sinon": "^4.5.0", "tap": "^16.3.7", diff --git a/packages/artillery/test/cloud-e2e/fargate/adot.test.js b/packages/artillery/test/cloud-e2e/fargate/adot.test.js new file mode 100644 index 0000000000..78aada6df8 --- /dev/null +++ b/packages/artillery/test/cloud-e2e/fargate/adot.test.js @@ -0,0 +1,122 @@ +'use strict'; + +const { test, afterEach, beforeEach } = require('tap'); +const { $ } = require('zx'); +const fs = require('fs'); +const { + generateTmpReportPath, + deleteFile, + getTestTags +} = require('../../cli/_helpers.js'); + +const { getDatadogSpans, getTestId } = require('./fixtures/adot/helpers.js'); + +//NOTE: all these tests report to Artillery Dashboard to dogfood and improve visibility +const baseTags = getTestTags(['type:acceptance']); + +let reportFilePath; +beforeEach(async (t) => { + reportFilePath = generateTmpReportPath(t.name, 'json'); +}); + +afterEach(async (t) => { + deleteFile(reportFilePath); +}); + +test('traces succesfully arrive to datadog', async (t) => { + // Arrange: + const apiKey = process.env.DD_TESTS_API_KEY; + const appKey = process.env.DD_TESTS_APP_KEY; + + if (!apiKey || !appKey) { + // Skipping test in case of running locally without DD keys + t.skip('Skipping test, missing Datadog API key or App key'); + } + + /// Expected values + const expectedTotalSpans = 52; // 4 VUs * (1 scenario root span + 2 requests + 10 timing zone spans (5 per request)) + const expectedVus = 4; + const expectedRequests = 8; + const expectedStatusCode200 = 8; + const expectedVusFailed = 0; + const tag = { key: 'testType', value: 'e2e' }; + + // Act: + const output = + await $`artillery run-fargate ${__dirname}/fixtures/adot/adot-dd-pass.yml --record --tags ${baseTags} --output ${reportFilePath}`; + + const testId = getTestId(output.stdout); + const report = JSON.parse(fs.readFileSync(reportFilePath, 'utf8')); + + let spanList; + try { + spanList = await getDatadogSpans( + apiKey, + appKey, + testId, + expectedTotalSpans + ); + } catch (err) { + t.fail('Error getting spans from Datadog: ' + err); + } + + const vuSpans = spanList.filter((span) => span.attributes.parent_id === '0'); + const requestSpans = spanList.filter( + (span) => span?.attributes?.resource_name === ('GET' || 'POST') + ); + + // Assert + t.equal(output.exitCode, 0, 'CLI Exit Code should be 0'); + t.equal( + spanList.length, + expectedTotalSpans, + `${expectedTotalSpans} spans in total should have arrived to Datadog` + ); + t.equal( + report.aggregate.counters['vusers.created'], + expectedVus, + `${expectedVus} VUs should have been created` + ); + t.equal( + vuSpans.length, + report.aggregate.counters['vusers.created'], + 'Num of traces (root spans) in Datadog should match num of vusers created in report' + ); + t.equal( + requestSpans.length, + expectedRequests, + `${expectedRequests} request spans should have arrived to Datadog` + ); + t.equal( + report.aggregate.counters['http.codes.200'], + expectedStatusCode200, + `Should have ${expectedStatusCode200} "200 OK" responses` + ); + t.equal( + requestSpans.filter( + (span) => span?.attributes?.custom?.http?.status_code === '200' + ).length, + report.aggregate.counters['http.codes.200'], + 'Num of request spans with status_code 200 in Datadog should match num of 200 OK responses in report' + ); + t.equal( + report.aggregate.counters['vusers.failed'], + expectedVusFailed, + `Should have ${expectedVusFailed} failed VUs` + ); + t.equal( + vuSpans.filter((span) => span.attributes.custom.error).length, + expectedVusFailed, + 'Num of traces with error should match failed VUs in report' + ); + t.hasProp( + requestSpans[0]?.attributes?.custom, + tag.key, + 'Request span should have the correct tag set from reporters config' + ); + t.equal( + requestSpans[0]?.attributes?.custom[tag.key], + tag.value, + 'Request span should have the correct tag value set from reporters config' + ); +}); diff --git a/packages/artillery/test/cloud-e2e/fargate/fixtures/adot/adot-dd-pass.yml b/packages/artillery/test/cloud-e2e/fargate/fixtures/adot/adot-dd-pass.yml new file mode 100644 index 0000000000..063c3208f2 --- /dev/null +++ b/packages/artillery/test/cloud-e2e/fargate/fixtures/adot/adot-dd-pass.yml @@ -0,0 +1,22 @@ +config: + target: "http://asciiart.artillery.io:8080" + phases: + - duration: 4 + arrivalRate: 1 + name: "Phase 1" + plugins: + publish-metrics: + - type: datadog + apiKey: "{{ $env.DD_TESTS_API_KEY }}" + traces: + serviceName: "adot-e2e" + tags: + - 'testType:e2e' + +scenarios: + - name: adot-e2e + flow: + - get: + url: "/dino" + - get: + url: "/pony" diff --git a/packages/artillery/test/cloud-e2e/fargate/fixtures/adot/helpers.js b/packages/artillery/test/cloud-e2e/fargate/fixtures/adot/helpers.js new file mode 100644 index 0000000000..7e002f462d --- /dev/null +++ b/packages/artillery/test/cloud-e2e/fargate/fixtures/adot/helpers.js @@ -0,0 +1,67 @@ +'use strict'; + +const sleep = require('../../../../helpers/sleep.js'); +const got = require('got'); + +module.exports = { + getTestId, + getDatadogSpans +}; + +function getTestId(outputString) { + const regex = /Test run ID: \S+/; + const match = outputString.match(regex); + return match[0].replace('Test run ID: ', ''); +} + +async function getDatadogSpans(apiKey, appKey, testId, expectedTotalSpans) { + const url = 'https://api.datadoghq.com/api/v2/spans/events/search'; + const headers = { + Accept: 'application/json', + 'Content-Type': 'application/json', + 'DD-API-KEY': apiKey, + 'DD-APPLICATION-KEY': appKey + }; + + const body = { + data: { + attributes: { + filter: { + from: 'now-15m', + query: '@test_id:' + testId, + to: 'now' + }, + options: { + timezone: 'GMT' + }, + page: { + limit: 500 + }, + sort: 'timestamp' + }, + type: 'search_request' + } + }; + + // Wait for spans to be available in Datadog + // Delay is 30s, to avoid hitting rate limits + const maxRetry = 12; + const delay = 30000; + + let spanList = []; + let retryNum = 0; + while (spanList.length < expectedTotalSpans && retryNum <= maxRetry) { + console.log(`ADOT Datadog test: Awaiting spans... (retry #${retryNum})`); + + spanList = await got + .post(url, { + headers: headers, + json: body + }) + .then((response) => JSON.parse(response.body).data); + await sleep(delay); + retryNum++; + } + + return spanList; +} diff --git a/packages/artillery/test/unit/fargate-bom.test.js b/packages/artillery/test/unit/fargate-bom.test.js new file mode 100644 index 0000000000..dd09bd2ecd --- /dev/null +++ b/packages/artillery/test/unit/fargate-bom.test.js @@ -0,0 +1,149 @@ +'use strict'; + +const { test, afterEach } = require('tap'); +const { applyScriptChanges } = require('../../lib/platform/aws-ecs/legacy/bom'); + +// TODO: Add tests for other functions in bom.js + +test('applyScriptChanges should resolve config templates with cli variables', async (t) => { + // Arrange + global.artillery.testRunId = 'bombolini_id_1234567890'; + const context = { + opts: { + scriptData: { + config: { + payload: { + path: '{{ fakePayloadPath }}' + }, + plugins: { + 'publish-metrics': [ + { + type: 'datadog', + apiKey: '{{ fakeApiKey }}', + traces: { + serviceName: '{{ fakeServiceName }}', + attributes: { + testId: '{{ $testId }}' + } + } + } + ] + } + } + }, + absoluteScriptPath: '/path/to/script.yml', + flags: { + variables: JSON.stringify({ + fakeServiceName: 'Bombolini', + fakePayloadPath: '/path/to/payload.json', + fakeApiKey: 'my_bombolini_key_1234567890' + }) + } + } + }; + + // Act + applyScriptChanges(context, (err, context) => { + if (err) { + return t.fail(err); + } + + // Assert + t.equal( + context.opts.scriptData.config.payload.path, + '/path/to/payload.json', + 'Should resolve config templates with cli variables' + ); + t.equal( + context.opts.scriptData.config.plugins['publish-metrics'][0].apiKey, + 'my_bombolini_key_1234567890', + 'Should resolve config templates with cli variables on all config depth levels' + ); + t.equal( + context.opts.scriptData.config.plugins['publish-metrics'][0].traces + .serviceName, + 'Bombolini', + 'Should resolve config templates with cli variables on all config depth levels' + ); + t.equal( + context.opts.scriptData.config.plugins['publish-metrics'][0].traces + .attributes.testId, + 'bombolini_id_1234567890', + 'Should resolve $testId with global.artillery.testRunId' + ); + }); + delete global.artillery.testRunId; +}); + +test('applyScriptChanges should resolve config templates with env variables', async (t) => { + // Arrange + process.env.FAKE_PATH_TO_PAYLOAD = '/path/to/payload.json'; + process.env.FAKE_DD_API_KEY = 'my_bombolini_key_1234567890'; + process.env.FAKE_TEST_ID = 'bombolini_id_1234567890'; + const context = { + opts: { + scriptData: { + config: { + payload: { + path: '{{ $env.FAKE_PATH_TO_PAYLOAD }}' + }, + plugins: { + 'publish-metrics': [ + { + type: 'datadog', + apiKey: '{{ $processEnvironment.FAKE_DD_API_KEY }}', + traces: { + serviceName: '{{ $environment.FAKE_SERVICE_NAME }}', + attributes: { + testId: '{{ $env.FAKE_TEST_ID }}' + } + } + } + ] + } + } + }, + absoluteScriptPath: '/path/to/script.yml', + flags: { + environment: { + FAKE_SERVICE_NAME: 'Bombolini' + } + } + } + }; + + // Act + applyScriptChanges(context, (err, context) => { + if (err) { + t.fail(err); + } + + //Assert + t.equal( + context.opts.scriptData.config.payload.path, + '/path/to/payload.json', + 'Should resolve $env templates with env vars' + ); + t.equal( + context.opts.scriptData.config.plugins['publish-metrics'][0].apiKey, + 'my_bombolini_key_1234567890', + 'Should resolve $processEnvironment templates with env vars' + ); + t.equal( + context.opts.scriptData.config.plugins['publish-metrics'][0].traces + .serviceName, + 'Bombolini', + 'Should resolve $environment templates with vars from flags.environment' + ); + t.equal( + context.opts.scriptData.config.plugins['publish-metrics'][0].traces + .attributes.testId, + 'bombolini_id_1234567890', + 'Should resolve env vars on all levels of test script' + ); + }); + + delete process.env.FAKE_PATH_TO_PAYLOAD; + delete process.env.FAKE_DD_API_KEY; + delete process.env.FAKE_TEST_ID; +});