configMap.yaml

---
apiVersion: v1
kind: ConfigMap
metadata:
  name: splunk-kubernetes-logging
  namespace: default
  labels:
    app: splunk-kubernetes-logging
    version: 1.5.4
data:
  fluent.conf: |-
    @include system.conf
    @include source.containers.conf
    @include source.files.conf
    @include source.journald.conf
    @include monit.conf
    @include output.conf
  system.conf: |-
    # system wide configurations
    <system>
      log_level info
      root_dir /tmp/fluentd
    </system>
  source.containers.conf: |-
    # This configuration file for Fluentd / td-agent is used
    # to watch changes to Docker log files. The kubelet creates symlinks that
    # capture the pod name, namespace, container name & Docker container ID
    # to the docker logs for pods in the /var/log/containers directory on the host.
    # If running this fluentd configuration in a Docker container, the /var/log
    # directory should be mounted in the container.
    # reading kubelet logs from journal
    #
    # Reference:
    # https://github.com/kubernetes/community/blob/20d2f6f5498a5668bae2aea9dcaf4875b9c06ccb/contributors/design-proposals/node/kubelet-cri-logging.md
    #
    # Json Log Example:
    # {"log":"[info:2016-02-16T16:04:05.930-08:00] Some log text here\n","stream":"stdout","time":"2016-02-17T00:04:05.931087621Z"}
    # CRI Log Example (not supported):
    # 2016-02-17T00:04:05.931087621Z stdout P { 'long': { 'json', 'object output' },
    # 2016-02-17T00:04:05.931087621Z stdout F 'splitted': 'partial-lines' }
    # 2016-02-17T00:04:05.931087621Z stdout F [info:2016-02-16T16:04:05.930-08:00] Some log text here
    <source>
      @id containers.log
      @type tail
      @label @CONCAT
      tag tail.containers.*
      path /var/log/containers/*.log
      pos_file /var/log/splunk-fluentd-containers.log.pos
      path_key source
      read_from_head true
      enable_stat_watcher true
      refresh_interval 60
      <parse>
        @type json
        time_format %Y-%m-%dT%H:%M:%S.%NZ
        time_key time
        time_type string
        localtime false
      </parse>
    </source>
  source.files.conf: |-
    # This fluentd conf file contains sources for log files other than container logs.
    <source>
      @id tail.file.kube-audit
      @type tail
      @label @CONCAT
      tag tail.file.kube:apiserver-audit
      path /var/log/kube-apiserver-audit.log
      pos_file /var/log/splunk-fluentd-kube-audit.pos
      read_from_head true
      path_key source
      <parse>
        @type regexp
        expression /^(?<log>.*)$/
        time_key time
        time_type string
        time_format %Y-%m-%dT%H:%M:%SZ
      </parse>
    </source>
  source.journald.conf: |-
    # This fluentd conf file contains configurations for reading logs from systemd journal.
    <source>
      @id journald-docker
      @type systemd
      @label @CONCAT
      tag journald.kube:docker
      path "/run/log/journal"
      matches [{ "_SYSTEMD_UNIT": "docker.service" }]
      read_from_head true
      <storage>
        @type local
        persistent true
        path /var/log/splunkd-fluentd-journald-docker.pos.json
      </storage>
      <entry>
        field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"}
        field_map_strict true
      </entry>
    </source>
    <source>
      @id journald-kubelet
      @type systemd
      @label @CONCAT
      tag journald.kube:kubelet
      path "/run/log/journal"
      matches [{ "_SYSTEMD_UNIT": "kubelet.service" }]
      read_from_head true
      <storage>
        @type local
        persistent true
        path /var/log/splunkd-fluentd-journald-kubelet.pos.json
      </storage>
      <entry>
        field_map {"MESSAGE": "log", "_SYSTEMD_UNIT": "source"}
        field_map_strict true
      </entry>
    </source>
  monit.conf: |-
    <source>
      @id fluentd-monitor-agent
      @type monitor_agent
      @label @PARSE
      bind 0.0.0.0
      port 24220
    </source>
  output.conf: |-
    #Events are emitted to the CONCAT label from the container, file and journald sources for multiline processing.
    <label @CONCAT>
      # = filters for container logs =
      <filter tail.containers.var.log.containers.dns-controller*dns-controller*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      <filter tail.containers.var.log.containers.kube-dns*sidecar*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      <filter tail.containers.var.log.containers.kube-dns*dnsmasq*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      <filter tail.containers.var.log.containers.kube-apiserver*kube-apiserver*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      <filter tail.containers.var.log.containers.kube-controller-manager*kube-controller-manager*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      <filter tail.containers.var.log.containers.kube-dns-autoscaler*autoscaler*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      <filter tail.containers.var.log.containers.kube-proxy*kube-proxy*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      <filter tail.containers.var.log.containers.kube-scheduler*kube-scheduler*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      <filter tail.containers.var.log.containers.kube-dns*kubedns*.log>
        @type concat
        key log
        timeout_label @PARSE
        stream_identity_key stream
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
        separator ""
        use_first_timestamp true
      </filter>
      # = filters for journald logs =
      <filter journald.kube:kubelet>
        @type concat
        key log
        timeout_label @PARSE
        multiline_start_regexp /^\w[0-1]\d[0-3]\d/
        flush_interval 5
      </filter>
      # Events are relabeled then emitted to the PARSE label for processing
      <match **>
        @type relabel
        @label @PARSE
      </match>
    </label>
    <label @PARSE>
      # filter to remove empty lines
      <filter tail.containers.**>
        @type grep
        <exclude>
         key log
         pattern \A\z
        </exclude>
      </filter>
      # Enrich log with k8s metadata
      <filter tail.containers.**>
        @type kubernetes_metadata
        annotation_match [ ".*" ]
        de_dot false
        watch true
        cache_ttl 3600
      </filter>
      <filter tail.containers.**>
        @type record_transformer
        enable_ruby
        <record>
          # set the sourcetype from splunk.com/sourcetype pod annotation or set it to kube:container:CONTAINER_NAME
          sourcetype ${record.dig("kubernetes", "annotations", "splunk.com/sourcetype") ? record.dig("kubernetes", "annotations", "splunk.com/sourcetype") : "kube:container:"+record.dig("kubernetes","container_name")}
          container_name ${record.dig("kubernetes","container_name")}
          namespace ${record.dig("kubernetes","namespace_name")}
          pod ${record.dig("kubernetes","pod_name")}
          container_id ${record.dig("docker","container_id")}
          pod_uid ${record.dig("kubernetes","pod_id")}
          container_image ${record.dig("kubernetes","container_image")}
          # set the cluster_name field to the configured value, or default to "cluster_name"
          cluster_name cluster_name
          # set the splunk_index field to the value found in the pod splunk.com/index annotations. if not set, use namespace annotation, or default to the default_index
          splunk_index ${record.dig("kubernetes", "annotations", "splunk.com/index."+record.dig("kubernetes","container_name")) ? record.dig("kubernetes", "annotations", "splunk.com/index."+record.dig("kubernetes","container_name")) : record.dig("kubernetes", "annotations", "splunk.com/index") ? record.dig("kubernetes", "annotations", "splunk.com/index") : record.dig("kubernetes", "namespace_annotations", "splunk.com/index") ? (record["kubernetes"]["namespace_annotations"]["splunk.com/index"]) : ("main")}
          label_app ${record.dig("kubernetes","labels","app")}
          label_k8s-app ${record.dig("kubernetes","labels","k8s-app")}
          label_release ${record.dig("kubernetes","labels","release")}
          exclude_list ${record.dig("kubernetes", "annotations", "splunk.com/exclude") ? record.dig("kubernetes", "annotations", "splunk.com/exclude") : record.dig("kubernetes", "namespace_annotations", "splunk.com/exclude") ? (record["kubernetes"]["namespace_annotations"]["splunk.com/exclude"]) : ("false")}
        </record>
      </filter>
      <filter tail.containers.**>
        # Exclude all logs that are marked
        @type grep
        <exclude>
          key exclude_list
          pattern /^true$/
        </exclude>
      </filter>
      # extract pod_uid and container_name for CRIO runtime

      # Add sourcetype for pod if provided in values file
      <filter tail.containers.var.log.containers.dns-controller*dns-controller*.log>
        @type record_transformer
        <record>
          sourcetype kube:dns-controller
        </record>
      </filter>
      <filter tail.containers.var.log.containers.kube-dns*sidecar*.log>
        @type record_transformer
        <record>
          sourcetype kube:kubedns-sidecar
        </record>
      </filter>
      <filter tail.containers.var.log.containers.kube-dns*dnsmasq*.log>
        @type record_transformer
        <record>
          sourcetype kube:dnsmasq
        </record>
      </filter>
      <filter tail.containers.var.log.containers.kube-apiserver*kube-apiserver*.log>
        @type record_transformer
        <record>
          sourcetype kube:kube-apiserver
        </record>
      </filter>
      <filter tail.containers.var.log.containers.kube-controller-manager*kube-controller-manager*.log>
        @type record_transformer
        <record>
          sourcetype kube:kube-controller-manager
        </record>
      </filter>
      <filter tail.containers.var.log.containers.kube-dns-autoscaler*autoscaler*.log>
        @type record_transformer
        <record>
          sourcetype kube:kube-dns-autoscaler
        </record>
      </filter>
      <filter tail.containers.var.log.containers.kube-proxy*kube-proxy*.log>
        @type record_transformer
        <record>
          sourcetype kube:kube-proxy
        </record>
      </filter>
      <filter tail.containers.var.log.containers.kube-scheduler*kube-scheduler*.log>
        @type record_transformer
        <record>
          sourcetype kube:kube-scheduler
        </record>
      </filter>
      <filter tail.containers.var.log.containers.kube-dns*kubedns*.log>
        @type record_transformer
        <record>
          sourcetype kube:kubedns
        </record>
      </filter>

      # create source and sourcetype
      <filter journald.**>
        @type jq_transformer
        jq '.record.source = "/run/log/journal/" + .record.source | .record.sourcetype = (.tag | ltrimstr("journald.")) | .record.cluster_name = "cluster_name" | .record.splunk_index = "main" |.record'
      </filter>

      # = filters for non-container log files =
      # extract sourcetype
      <filter tail.file.**>
        @type jq_transformer
        jq '.record.sourcetype = (.tag | ltrimstr("tail.file.")) | .record.cluster_name = "cluster_name" | .record.splunk_index = "main" | .record'
      </filter>
      # = custom filters specified by users =
      # Events are relabeled then emitted to the SPLUNK label for ingestion
      <match **>
        @type relabel
        @label @SPLUNK
      </match>
    </label>
    <label @SPLUNK>
      # = output =
      <match **>
        @type splunk_hec
        protocol https
        hec_host "MY-SPLUNK-HOST"
        consume_chunk_on_4xx_errors true
        hec_port 8088
        hec_token "#{ENV['SPLUNK_HEC_TOKEN']}"
        index_key splunk_index
        insecure_ssl true
        host "#{ENV['K8S_NODE_NAME']}"
        source_key source
        sourcetype_key sourcetype
        <fields>
          # currently CRI does not produce log paths with all the necessary
          # metadata to parse out pod, namespace, container_name, container_id.
          # this may be resolved in the future by this issue: https://github.com/kubernetes/kubernetes/issues/58638#issuecomment-385126031
          container_image
          pod_uid
          pod
          container_name
          namespace
          container_id
          cluster_name
          label_app
          label_k8s-app
          label_release
        </fields>
        app_name splunk-kubernetes-logging
        app_version 1.5.4
        <buffer index>
          @type memory
          chunk_limit_records 100000
          chunk_limit_size 20m
          flush_interval 5s
          flush_thread_count 1
          overflow_action block
          retry_max_times 10
          retry_type periodic
          retry_wait 30
          total_limit_size 600m
        </buffer>
        <format>
          # we just want to keep the raw logs, not the structure created by docker or journald
          @type single_value
          message_key log
          add_newline false
        </format>
      </match>
    </label>