Standardize DD_TAGS format (#162)

Co-authored-by: May Lee <mayl@alumni.cmu.edu> Co-authored-by: Sarah Witt <sarah.witt@datadoghq.com>
DataDog · Jul 21, 2023 · 636b5e9 · 636b5e9
1 parent 3fcdd20
commit 636b5e9
Show file tree

Hide file tree

Showing 10 changed files with 200 additions and 107 deletions.
diff --git a/README.md b/README.md
@@ -37,6 +37,27 @@ env:
   DD_API_KEY: <DATADOG_API_KEY>
 ```
 
+#### Assigning Tags
+
+For an overview about tags, read [Getting Started with Tags](https://docs.datadoghq.com/getting_started/tagging/).
+
+Custom tags can be configured with the environment variable `DD_TAGS`. These tags will be attached to the application logs, metrics, and traces as span tags.
+
+By default, `DD_TAGS` is expected to be a comma separated list of tags.
+
+```yaml
+env:
+  DD_TAGS: "key1:value1,key2:value2,key3:value3"
+```
+
+To use a different separator, set `DD_TAGS_SEP` to the desired separator.
+
+```yaml
+env:
+  DD_TAGS: "key1:value1 key2:value2 key3:value3"
+  DD_TAGS_SEP: " "
+```
+
 ### Instrument your application
 
 Instrument your application to send custom metrics and APM traces through DogStatsD and the Datadog Trace Agent.

diff --git a/bin/compile b/bin/compile
@@ -31,9 +31,11 @@ fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
-cp "${ROOT_DIR}/lib/scripts/parse_env_vars.rb" "${DATADOG_DIR}/scripts/parse_env_vars.rb"
+cp "${ROOT_DIR}/lib/scripts/update_datadog_config.rb" "${DATADOG_DIR}/scripts/update_datadog_config.rb"
+
 cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
+cp "${ROOT_DIR}/lib/scripts/parse_env_vars.rb" "${DATADOG_DIR}/scripts/parse_env_vars.rb"
 cp "${ROOT_DIR}/lib/scripts/nc.rb" "${DATADOG_DIR}/scripts/nc.rb"
 cp "${ROOT_DIR}/lib/scripts/utils.sh" "${DATADOG_DIR}/scripts/utils.sh"
 cp "${ROOT_DIR}/lib/scripts/check_datadog.sh" "${DATADOG_DIR}/scripts/check_datadog.sh"
@@ -59,18 +61,14 @@ chmod +x "${BUILD_DIR}/.profile.d/00-test-endpoint.sh"
 chmod +x "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
 chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 
-# export DD environment variables
+# import helper functions
 . "${DATADOG_DIR}/scripts/utils.sh"
+
+# export DD environment variables
 dd_export_env "${DATADOG_DIR}/.raw_datadog_env"
 
 # sanitize env vars and export a new a env file
 ruby "${DATADOG_DIR}/scripts/parse_env_vars.rb" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
 
 # export DD_TAGS for ddtrace
-DD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-export DD_TAGS
-DD_DOGSTATSD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-export DD_DOGSTATSD_TAGS
-
-# mark the script as finished, useful to sync the update_agent_config script
-touch "${DATADOG_DIR}/.setup_completed"
+export DD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
diff --git a/bin/supply b/bin/supply
@@ -31,6 +31,8 @@ fi
 cp "${ROOT_DIR}/lib/trace-agent" "${DATADOG_DIR}/trace-agent"
 
 cp "${ROOT_DIR}/lib/scripts/create_logs_config.rb" "${DATADOG_DIR}/scripts/create_logs_config.rb"
+cp "${ROOT_DIR}/lib/scripts/update_datadog_config.rb" "${DATADOG_DIR}/scripts/update_datadog_config.rb"
+
 cp "${ROOT_DIR}/lib/scripts/update_tags.rb" "${DATADOG_DIR}/scripts/update_tags.rb"
 
 cp "${ROOT_DIR}/lib/scripts/parse_env_vars.rb" "${DATADOG_DIR}/scripts/parse_env_vars.rb"
@@ -59,18 +61,14 @@ chmod +x "${BUILD_DIR}/.profile.d/00-test-endpoint.sh"
 chmod +x "${BUILD_DIR}/.profile.d/02-redirect-logs.sh"
 chmod +x "${BUILD_DIR}/.profile.d/01-run-datadog.sh"
 
-# export DD environment variables
+# import helper functions
 . "${DATADOG_DIR}/scripts/utils.sh"
+
+# export DD environment variables
 dd_export_env "${DATADOG_DIR}/.raw_datadog_env"
 
 # sanitize env vars and export a new a env file
 ruby "${DATADOG_DIR}/scripts/parse_env_vars.rb" "${DATADOG_DIR}/.raw_datadog_env" "${DATADOG_DIR}/.datadog_env"
 
 # export DD_TAGS for ddtrace
-DD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-export DD_TAGS
-DD_DOGSTATSD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-export DD_DOGSTATSD_TAGS
-
-# mark the script as finished, useful to sync the update_agent_config script
-touch "${DATADOG_DIR}/.setup_completed"
+export DD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
diff --git a/lib/run-datadog.sh b/lib/run-datadog.sh
@@ -9,11 +9,9 @@ SUPPRESS_DD_AGENT_OUTPUT="${SUPPRESS_DD_AGENT_OUTPUT:-true}"
 DD_ENABLE_CAPI_METADATA_COLLECTION="${DD_ENABLE_CAPI_METADATA_COLLECTION:-false}"
 LOCKFILE="${DATADOG_DIR}/lock"
 FIRST_RUN="${FIRST_RUN:-true}"
-USER_TAGS="${DD_TAGS}"
-DD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-export DD_TAGS
-DD_DOGSTATSD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-export DD_DOGSTATSD_TAGS
+
+export DD_TAGS=$(ruby "${DATADOG_DIR}/scripts/get_tags.rb")
+echo "${DD_TAGS}" > "${DATADOG_DIR}/.dd_tags.txt"
 
 source "${DATADOG_DIR}/scripts/utils.sh"
 
@@ -97,10 +95,6 @@ setup_datadog() {
     # Create folder for storing PID files
     mkdir run
 
-
-    # DSD requires its own config file
-    cp dist/datadog.yaml dist/dogstatsd.yaml
-
     if [ -a ./agent ] && { [ "${DD_LOGS_ENABLED}" = "true" ] || [ "${DD_ENABLE_CHECKS}" = "true" ]; }; then
       if [ "${DD_LOGS_ENABLED}" = "true" -a "${DD_LOGS_VALID_ENDPOINT}" = "false" ]; then
         echo "Log endpoint not valid, not starting agent"
@@ -114,14 +108,18 @@ setup_datadog() {
       sed -i "s~log_file: AGENT_LOG_FILE~log_file: ${DD_LOG_FILE}~" dist/datadog.yaml
     fi
   popd
+
+  # update datadog config
+  ruby "${DATADOG_DIR}/scripts/update_datadog_config.rb"
+
+  # mark the script as finished, useful to sync the update_agent_config script
+  touch "${DATADOG_DIR}/.setup_completed"
+
 }
 
 start_datadog() {
-  DD_TAGS="${USER_TAGS}"
-  DD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-  export DD_TAGS
-  DD_DOGSTATSD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-  export DD_DOGSTATSD_TAGS
+  export DD_TAGS=$(ruby "${DATADOG_DIR}/scripts/get_tags.rb")
+
   pushd "${DATADOG_DIR}"
     export DD_LOG_FILE="${DATADOG_DIR}/dogstatsd.log"
     export DD_API_KEY
@@ -154,27 +152,27 @@ start_datadog() {
 
         echo "Starting Datadog agent"
         if [ "${SUPPRESS_DD_AGENT_OUTPUT}" = "true" ]; then
-          ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
+          env -u DD_TAGS ./agent run --cfgpath dist/ --pidfile run/agent.pid > /dev/null 2>&1 &
         else
-          ./agent run --cfgpath dist/ --pidfile run/agent.pid &
+          env -u DD_TAGS ./agent run --cfgpath dist/ --pidfile run/agent.pid &
         fi
       fi
     else
       echo "Starting dogstatsd agent"
       export DD_LOG_FILE=dogstatsd.log
       if [ "${SUPPRESS_DD_AGENT_OUTPUT}" = "true" ]; then
-        ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
+        env -u DD_TAGS ./dogstatsd start --cfgpath dist/ > /dev/null 2>&1 &
       else
-        ./dogstatsd start --cfgpath dist/ &
+        env -u DD_TAGS ./dogstatsd start --cfgpath dist/ &
       fi
       echo $! > run/dogstatsd.pid
     fi
     if [ "${FIRST_RUN}" = "true" ]; then
       echo "Starting trace agent"
       if [ "${SUPPRESS_DD_AGENT_OUTPUT}" = "true" ]; then
-        ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
+        env -u DD_TAGS ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid > /dev/null 2>&1 &
       else
-        ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid &
+        env -u DD_TAGS ./trace-agent --config dist/datadog.yaml --pid run/trace-agent.pid &
       fi
       FIRST_RUN=false
     fi

diff --git a/lib/scripts/create_logs_config.rb b/lib/scripts/create_logs_config.rb
@@ -6,13 +6,12 @@
 
 require 'json'
 
-dd_env_file = "/home/vcap/app/.datadog/.sourced_env_datadog"
 node_agent_tags = "/home/vcap/app/.datadog/node_agent_tags.txt"
 
 logs_config_dir = ENV['LOGS_CONFIG_DIR']
 logs_config = ENV['LOGS_CONFIG']
 dd_tags = ENV['DD_TAGS']
-dd_node_agent_tags = ENV['DD_NODE_AGENTS_TAGS'] || (File.file?(node_agent_tags) ? File.read(node_agent_tags) : "")
+dd_node_agent_tags = ENV['DD_NODE_AGENT_TAGS'] || (File.file?(node_agent_tags) ? File.read(node_agent_tags) : "")
 
 def sanitize(tags_env_var, separator)
   tags_list = tags_env_var.gsub(",\"", ";\"").split(separator)
@@ -42,7 +41,7 @@ def sanitize(tags_env_var, separator)
   if !dd_node_agent_tags.nil?
     tags_list += sanitize(dd_node_agent_tags, ",")
   else
-    puts "Could not find DD_NODE_AGENTS_TAGS env var"
+    puts "Could not find DD_NODE_AGENT_TAGS env var"
   end
 
   if !tags_list.empty?

diff --git a/lib/scripts/get_tags.rb b/lib/scripts/get_tags.rb
@@ -4,16 +4,19 @@
 
 require 'json'
 
-# if DD_TAGS[0] is comma or space, then set is as delimiter
-# else continue as usual
+NODE_AGENT_TAGS_FILE = "/home/vcap/app/.datadog/node_agent_tags.txt"
 
 def parse_tags(tags)
-  delimiter = ','
-  delimiter = ' ' if tags.count(' ') > tags.count(',')
   begin
+    delimiter = ','
+    delimiter = ' ' if tags.count(' ') > tags.count(',')
+    if !ENV["DD_TAGS_SEP"].nil?
+      delimiter = ENV["DD_TAGS_SEP"]
+    end
     return tags.split(delimiter)
   rescue Exception => e
-    puts "there was an issue parsing the tags in #{tags.__name__}: #{e}"
+    puts "there was an issue parsing the tags in '#{tags}': #{e.message}"
+    return []
   end
 end
 
@@ -38,7 +41,17 @@ def parse_tags(tags)
   # we do this to separate commas inside json values from tags separator commas
   node_agent_tags = node_agent_tags.gsub(",\"", ";\"")
   all_node_agent_tags = parse_tags(node_agent_tags)
-  tags += all_node_agent_tags.reject { |tag| tag.include?(';') }
+  if !all_node_agent_tags.empty?
+    tags += all_node_agent_tags.keep_if { |tag| !tag.include?(';') }
+  end
+end
+
+node_agent_tags_file = File.file?(NODE_AGENT_TAGS_FILE) ? File.read(NODE_AGENT_TAGS_FILE).strip : nil
+if node_agent_tags_file
+  node_agent_tags = parse_tags(node_agent_tags_file)
+  if !node_agent_tags.empty?
+    tags += node_agent_tags
+  end
 end
 
 vcap_variables.each do |vcap_var_name|
@@ -63,7 +76,7 @@ def parse_tags(tags)
     user_tags = parse_tags(user_tags)
     tags += user_tags
   rescue Exception => e
-    puts "there was an issue parsing the tags in TAGS: #{e}"
+    puts "there was an issue parsing the tags in TAGS: #{e.message}"
   end
 end
 
@@ -73,21 +86,17 @@ def parse_tags(tags)
     user_tags = parse_tags(user_tags)
     tags += user_tags
   rescue Exception => e
-    puts "there was an issue parsing the tags in DD_TAGS: #{e}"
+    puts "there was an issue parsing the tags in DD_TAGS: #{e.message}"
   end
 end
 
 version_file = '/home/vcap/app/.datadog/VERSION'
 if File.exist?(version_file)
-  buildpack_version = File.open(version_file, 'r') { |file| file.read.chomp }
-  tags << "buildpack_version:#{buildpack_version}"
+  buildpack_version = File.open(version_file, 'r') { |file| file.read.strip }
+  tags << "buildpack_version:#{buildpack_version.strip}"
 end
 
-tags = tags.map { |tag| tag.gsub(' ', '_') }.uniq
+tags = tags.map { |tag| tag.gsub(' ', '_') }
+tags = tags.uniq
 
-legacy_tags = ENV['LEGACY_TAGS_FORMAT'] || false
-if legacy_tags
-  puts tags.join(',')
-else
-  puts tags.join(' ')
-end
+puts tags.join(',')
diff --git a/lib/scripts/update_agent_config.sh b/lib/scripts/update_agent_config.sh
@@ -14,34 +14,36 @@ release_lock() {
 }
 
 write_tags_to_file() {
-    # combine DD_TAGS and DD_NODE_AGENT_TAGS into DD_TAGS
-    DD_TAGS=$(LEGACY_TAGS_FORMAT=true ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-    export DD_TAGS
-    DD_DOGSTATSD_TAGS=$(LEGACY_TAGS_FORMAT=true ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
-    export DD_DOGSTATSD_TAGS
+    export DD_TAGS=$(ruby "${DATADOG_DIR}"/scripts/get_tags.rb)
+
     export LOGS_CONFIG_DIR="${DATADOG_DIR}/dist/conf.d/logs.d"
     export LOGS_CONFIG
 
-    # update logs configs with the new tags
-    if [ -n "${LOGS_CONFIG}" ] && [ "${DD_ENABLE_CAPI_METADATA_COLLECTION}" = "true" ]; then
-        mkdir -p "${LOGS_CONFIG_DIR}"
-        log_info "Updating logs config"
-        ruby "${DATADOG_DIR}/scripts/create_logs_config.rb"
-    fi
-
     log_info "Updating node_agent_tags.txt"
     ruby "${DATADOG_DIR}/scripts/update_tags.rb"
 
+    # update datadog config
+    ruby "${DATADOG_DIR}/scripts/update_datadog_config.rb"
+
+    if [ "${DD_ENABLE_CAPI_METADATA_COLLECTION}" = "true" ]; then
+        # update logs configs
+        if [ -n "${LOGS_CONFIG}" ]; then
+            mkdir -p "${LOGS_CONFIG_DIR}"
+            log_info "Updating logs config"
+            ruby "${DATADOG_DIR}/scripts/create_logs_config.rb"
+        fi
+    fi
+
     # log DD_TAGS and DD_NODE_AGENT_TAGS values
     log_debug "node_agent_tags.txt=$(cat "${DATADOG_DIR}"/node_agent_tags.txt)"
     log_debug "(AFTER)DD_NODE_AGENT_TAGS=${DD_NODE_AGENT_TAGS}"
-    log_debug "DD_DOGSTATSD_TAGS=${DD_DOGSTATSD_TAGS}"
+    log_debug "DD_TAGS=${DD_TAGS}"
 }
 
 main() {
     # source relevant DD tags
     while ! [ -f "${DATADOG_DIR}/.setup_completed" ]; do
-        echo "Supply script not completed, waiting ..."
+        echo "Datadog setup is not completed, waiting ..."
         sleep 1
     done