out_datadog: support configurable hostname

[sudomateo]

Is your feature request related to a problem? Please describe.

The Datadog /api/v2/logs API accepts a request body that contains the hostname attribute, like so.

[
  {
    "ddsource": "nginx",
    "ddtags": "env:staging,version:5.1",
    "hostname": "i-012345678",
    "message": "2019-11-19T14:37:58,995 INFO [process.name][20081] Hello World",
    "service": "payment"
  }
]

However when Fluent Bit sends logs to Datadog it sets all of those attributes except hostname.

fluent-bit/plugins/out_datadog/datadog.h

Lines 30 to 33 in 5ce9c43

	#define FLB_DATADOG_DD_SOURCE_KEY "ddsource"
	#define FLB_DATADOG_DD_SERVICE_KEY "service"
	#define FLB_DATADOG_DD_TAGS_KEY "ddtags"
	#define FLB_DATADOG_DD_MESSAGE_KEY "message"

This means that logs sent from Fluent Bit to Datadog will not be associated with a host. This is an issue because operators cannot query logs by host nor can they navigate from the infrastructure list of hosts to the logs for a particular host.

Describe the solution you'd like

Fluent Bit should support specifying the hostname attribute similar to how it already supports dd_source and dd_service. A trivial diff of what this would look like is included below.

diff --git a/plugins/out_datadog/datadog.c b/plugins/out_datadog/datadog.c
index c92992f53..f2e9f8a19 100644
--- a/plugins/out_datadog/datadog.c
+++ b/plugins/out_datadog/datadog.c
@@ -247,6 +247,14 @@ static int datadog_format(struct flb_config *config,
                                           ctx->dd_service, flb_sds_len(ctx->dd_service));
         }

+        /* hostname */
+        if (ctx->hostname != NULL) {
+            dd_msgpack_pack_key_value_str(&mp_pck,
+                                          FLB_DATADOG_DD_HOSTNAME_KEY,
+                                          sizeof(FLB_DATADOG_DD_HOSTNAME_KEY) -1,
+                                          ctx->hostname, flb_sds_len(ctx->hostname));
+        }
+
         /* Append initial object k/v */
         ind = 0;
         for (i = 0; i < map_size; i++) {
@@ -508,6 +516,11 @@ static struct flb_config_map config_map[] = {
      "(e.g. 'postgres' or 'nginx'). If unset, Datadog will expect the source "
      "to be set as the `ddsource` attribute."
     },
+    {
+     FLB_CONFIG_MAP_STR, "hostname", NULL,
+     0, FLB_TRUE, offsetof(struct flb_out_datadog, hostname),
+     "Hostname that emitted logs should be associated with."
+    },
     {
      FLB_CONFIG_MAP_STR, "dd_tags", NULL,
      0, FLB_TRUE, offsetof(struct flb_out_datadog, dd_tags),
diff --git a/plugins/out_datadog/datadog.h b/plugins/out_datadog/datadog.h
index 15856be39..f3b271c4a 100644
--- a/plugins/out_datadog/datadog.h
+++ b/plugins/out_datadog/datadog.h
@@ -29,6 +29,7 @@
 #define FLB_DATADOG_DEFAULT_TAG_KEY   "tagkey"
 #define FLB_DATADOG_DD_SOURCE_KEY     "ddsource"
 #define FLB_DATADOG_DD_SERVICE_KEY    "service"
+#define FLB_DATADOG_DD_HOSTNAME_KEY   "hostname"
 #define FLB_DATADOG_DD_TAGS_KEY       "ddtags"
 #define FLB_DATADOG_DD_MESSAGE_KEY    "message"
 #define FLB_DATADOG_DD_LOG_KEY        "log"
@@ -65,6 +66,7 @@ struct flb_out_datadog {
     int nb_additional_entries;
     flb_sds_t dd_source;
     flb_sds_t dd_service;
+    flb_sds_t hostname;
     flb_sds_t dd_tags;
     flb_sds_t dd_message_key;

Describe alternatives you've considered

There are 2 workarounds that operators can follow to associate logs with a host. One is rather non-functional and the other is a bit hidden from the average operator.

First, operators can explicitly set the host tag using the dd_tags configuration option, like so.

  outputs:
  - name: datadog
    match: "*"
    # ...
    dd_tags: host:foobar

This will have logs show up with the host populated in Datadog but operators will still be unable to query logs by host nor can they navigate from the infrastructure list of hosts to the logs for a particular host. In my opinion this approach is a non-starter.

Second, operators can ensure their Fluent Bit records contain the field hostname.

  filters:
  - name: modify
    match: "*"
    # ...
    set:
    - hostname foobar

This will have logs show up with the host populated in Datadog and operators will finally be able to query logs by host and navigate from the infrastructure list of hosts to the logs for a particular host.

This approach works because Fluent Bit takes the entire MessagePack object, converts it to JSON, and sends it as the request body to the Datadog API. However, it's a bit hidden from operators and too "magical" to rely on.

fluent-bit/plugins/out_datadog/datadog.c

Lines 321 to 335 in 5ce9c43

	/* Convert from msgpack to JSON */
	out_buf = flb_msgpack_raw_to_json_sds(mp_sbuf.data, mp_sbuf.size);
	msgpack_sbuffer_destroy(&mp_sbuf);
	if (!out_buf) {
	flb_plg_error(ctx->ins, "error formatting JSON payload");
	if (remapped_tags) {
	flb_sds_destroy(remapped_tags);
	}
	flb_log_event_decoder_destroy(&log_decoder);
	return -1;
	}
	*out_data = out_buf;
	*out_size = flb_sds_len(out_buf);

Additional context

This additional context section contains Fluent Bit configuration and a screenshot of the resulting behavior in Datadog.

pipeline:
  inputs:
  - name: dummy
    tag: dummy
    dummy: |
      {
        "message": "From Fluent Bit",
        "test": "default"
      }

  outputs:
  - name: datadog
    match: "*"
    host: http-intake.logs.us5.datadoghq.com
    tls: off
    compress: gzip
    apikey: ${DD_API_KEY}
    dd_source: sudomateo
    dd_service: sudomateo
    dd_tags: env:development

pipeline:
  inputs:
  - name: dummy
    tag: dummy
    dummy: |
      {
        "message": "From Fluent Bit",
        "test": "with_host_tag"
      }

  outputs:
  - name: datadog
    match: "*"
    host: http-intake.logs.us5.datadoghq.com
    tls: off
    compress: gzip
    apikey: ${DD_API_KEY}
    dd_source: sudomateo
    dd_service: sudomateo
    dd_tags: env:development,host:foobar

pipeline:
  inputs:
  - name: dummy
    tag: dummy
    dummy: |
      {
        "message": "From Fluent Bit",
        "hostname": "foobar",
        "test": "with_hostname_in_record"
      }

  outputs:
  - name: datadog
    match: "*"
    host: http-intake.logs.us5.datadoghq.com
    tls: off
    compress: gzip
    apikey: ${DD_API_KEY}
    dd_source: sudomateo
    dd_service: sudomateo
    dd_tags: env:development