feat(fargate): use adot image for otel (#2481)

packages/artillery-plugin-publish-metrics/index.js

@@ -6,6 +6,11 @@ const NS = 'plugin:publish-metrics';
 const debug = require('debug')(NS);
 const A = require('async');
 
+const {
+  getADOTRelevantReporterConfigs,
+  resolveADOTConfigSettings
+} = require('./lib/open-telemetry/translators/vendor-adot');
+
 // List of reporters that use OpenTelemetry
 const REPORTERS_USING_OTEL = [
   'open-telemetry',
@@ -16,7 +21,9 @@ const REPORTERS_USING_OTEL = [
 ];
 module.exports = {
   Plugin,
-  LEGACY_METRICS_FORMAT: false
+  LEGACY_METRICS_FORMAT: false,
+  getADOTRelevantReporterConfigs,
+  resolveADOTConfigSettings
 };
 
 function Plugin(script, events) {

packages/artillery-plugin-publish-metrics/lib/open-telemetry/index.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const vendorTranslators = require('./vendor-translators');
+const { vendorTranslators } = require('./translators/vendor-otel');
 const {
   diag,
   DiagConsoleLogger,
@@ -71,6 +71,7 @@ class OTelReporter {
     if (this.tracesConfig) {
       global.artillery.OTEL_TRACING_ENABLED = true;
     }
+
     // Warn if traces are configured in multiple reporters
     this.warnIfDuplicateTracesConfigured(this.translatedConfigsList);
 
@@ -156,6 +157,11 @@ class OTelReporter {
     if (!this.metricsConfig && !this.tracesConfig) {
       return done();
     }
+
+    // Waiting for flush period to complete here rather than in trace/metric reporters
+    this.debug('Waiting for flush period to end');
+    await new Promise((resolve) => setTimeout(resolve, 10000));
+
     if (this.metricReporter) {
       await this.metricReporter.cleanup();
     }

packages/artillery-plugin-publish-metrics/lib/open-telemetry/tracing/base.js

@@ -178,8 +178,6 @@ class OTelTraceBase {
     }
 
     this.debug('Pending traces done');
-    this.debug('Waiting for flush period to complete');
-    await sleep(5000);
   }
 }
 

packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-adot.js

@@ -0,0 +1,150 @@
+'use strict';
+
+const ADOTSupportedTraceReporters = ['datadog'];
+const ADOTSupportedMetricReporters = [];
+
+// Getting the relevant reporter configurations from full publish-metrics configuration
+
+function getADOTRelevantReporterConfigs(publishMetricsConfig) {
+  const configs = publishMetricsConfig.filter(
+    (reporterConfig) =>
+      (ADOTSupportedTraceReporters.includes(reporterConfig.type) &&
+        reporterConfig.traces) ||
+      (ADOTSupportedMetricReporters.includes(reporterConfig.type) &&
+        reporterConfig.metrics)
+  );
+
+  return configs;
+}
+
+// Resolve the configuration settings for ADOT
+
+function resolveADOTConfigSettings(options) {
+  try {
+    const adotConfig = getADOTConfig(options.configList); // options.configList ( array of those reporter configurations from publish-metrics config that require ADOT )
+    const adotEnvVars = getADOTEnvVars(options.configList, options.dotenv); // options.dotenv (object with environment variables from user provided dotenv file)
+    return { adotConfig, adotEnvVars };
+  } catch (err) {
+    throw new Error(err);
+  }
+}
+
+// Assembling the configuration for ADOT (in OTel Collector format)
+
+function getADOTConfig(adotRelevantConfigs) {
+  const translatedVendorConfigs = adotRelevantConfigs.map((config) =>
+    vendorToCollectorConfigTranslators[config.type](config)
+  );
+
+  // Different vendors can be used for metrics and tracing so we need to merge configs from each vendor into one collector config
+  const finalADOTConfig = JSON.parse(JSON.stringify(collectorConfigTemplate));
+
+  translatedVendorConfigs.forEach((config) => {
+    finalADOTConfig.processors = Object.assign(
+      finalADOTConfig.processors,
+      config.processors
+    );
+    finalADOTConfig.exporters = Object.assign(
+      finalADOTConfig.exporters,
+      config.exporters
+    );
+    finalADOTConfig.service.pipelines = Object.assign(
+      finalADOTConfig.service.pipelines,
+      config.service.pipelines
+    );
+  });
+  return finalADOTConfig;
+}
+
+const collectorConfigTemplate = {
+  receivers: {
+    otlp: {
+      protocols: {
+        http: {
+          endpoint: '0.0.0.0:4318'
+        },
+        grpc: {
+          endpoint: '0.0.0.0:4317'
+        }
+      }
+    }
+  },
+  processors: {},
+  exporters: {},
+  service: {
+    pipelines: {}
+  }
+};
+
+// Map of functions that translate vendor-specific configuration to OpenTelemetry Collector configuration to be used by ADOT
+const vendorToCollectorConfigTranslators = {
+  datadog: (config) => {
+    const collectorConfig = JSON.parse(JSON.stringify(collectorConfigTemplate));
+    if (config.traces) {
+      collectorConfig.processors['batch/trace'] = {
+        timeout: '10s',
+        send_batch_max_size: 1024,
+        send_batch_size: 200
+      };
+      collectorConfig.exporters['datadog/api'] = {
+        traces: {
+          trace_buffer: 100
+        },
+        api: {
+          key: '${env:DD_API_KEY}'
+        }
+      };
+      collectorConfig.service.pipelines.traces = {
+        receivers: ['otlp'],
+        processors: ['batch/trace'],
+        exporters: ['datadog/api']
+      };
+    }
+    return collectorConfig;
+  }
+};
+
+// Handling vendor specific environment variables needed for ADOT configuration  (e.g. Authentication keys/tokens that can be provided in the script )
+
+function getADOTEnvVars(adotRelevantconfigs, dotenv) {
+  const envVars = {};
+  try {
+    adotRelevantconfigs.forEach((config) => {
+      const vendorVars = vendorSpecificEnvVarsForCollector[config.type](
+        config,
+        dotenv
+      );
+      Object.assign(envVars, vendorVars);
+    });
+  } catch (err) {
+    // We warn here instead of throwing because in the future we will support providing these variables through secrets
+    console.warn(err.message);
+  }
+  return envVars;
+}
+
+const vendorSpecificEnvVarsForCollector = {
+  datadog: (config, dotenv) => {
+    const apiKey = config.apiKey || dotenv?.DD_API_KEY;
+    // We validate API key here for Datadog (for now) because it is only required if Datadog tracing is set with test running on Fargate. (for local runs user configures their own agent, and for metrics if apiKey is not provided the reporter defaults to sending data to agent)
+    if (!apiKey) {
+      throw new Error(
+        "Datadog reporter Error: Missing Datadog API key. Provide it under 'apiKey' setting in your script or under 'DD_API_KEY' environment variable set in your dotenv file."
+      );
+    }
+    return { DD_API_KEY: apiKey };
+  }
+};
+
+module.exports = {
+  getADOTRelevantReporterConfigs,
+  resolveADOTConfigSettings,
+  // All func and vars below exported for testing purposes
+  getADOTEnvVars,
+  vendorSpecificEnvVarsForCollector,
+  getADOTConfig,
+  vendorToCollectorConfigTranslators,
+  ADOTSupportedTraceReporters,
+  ADOTSupportedMetricReporters,
+  collectorConfigTemplate
+};

packages/artillery-plugin-publish-metrics/lib/open-telemetry/vendor-translators.js → packages/artillery-plugin-publish-metrics/lib/open-telemetry/translators/vendor-otel.js

@@ -1,6 +1,6 @@
 'use strict';
 
-// Map of functions that translate vendor-specific configuration to OpenTelemetry configuration
+// Map of functions that translate vendor-specific reporter configuration to OpenTelemetry reporter configuration
 const vendorTranslators = {
   honeycomb: (config) => {
     if (config.enabled === false) {
@@ -115,4 +115,6 @@ function attributeListToObject(attributeList, reporterType) {
   return attributes;
 }
 
-module.exports = vendorTranslators;
+module.exports = {
+  vendorTranslators
+};