Add fileset kubernetes for coredns module to support Kubernetes deplo…

…yment

x-pack/filebeat/filebeat.reference.yml

@@ -89,7 +89,7 @@ filebeat.modules:
     enabled: true
   # Fileset for Kubernetes deployment
   kubernetes:
-    enabled: false
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.

x-pack/filebeat/module/coredns/_meta/config.yml

@@ -4,7 +4,7 @@
     enabled: true
   # Fileset for Kubernetes deployment
   kubernetes:
-    enabled: false
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.

x-pack/filebeat/module/coredns/_meta/fields.yml

@@ -58,7 +58,7 @@
       description: >
         dnssec flag
 
-  - name: kubernetes.replicaset.name
-    type: keyword
+  - name: kubernetes.replicaset
+    type: object
     description: >
-      Name of Kubernetes replica set
+      Kubernetes replica set

x-pack/filebeat/module/coredns/kubernetes/config/coredns.yml

@@ -0,0 +1,15 @@
+type: log
+paths:
+{{ range $i, $path := .paths }}
+  - {{$path}}
+{{ end }}
+tags: {{.tags}}
+json.keys_under_root: true
+processors:
+  - dissect:
+      tokenizer: "%{timestamp} [%{coredns.logging_level}] %{source.address}:%{source.port} - %{coredns.id} \"%{coredns.query.type} %{coredns.query.class} %{coredns.query.name} %{network.transport} %{coredns.query.size} %{coredns.dnssec_ok} %{bufsize}\" %{coredns.response.code} %{coredns.response.flags} %{coredns.response.size} %{temp.duration}s"
+      field: "message"
+      target_prefix: ""
+
+  - drop_fields: 
+      fields: [bufsize, time]

x-pack/filebeat/module/coredns/kubernetes/ingest/pipeline.json

@@ -0,0 +1,65 @@
+{
+  "description": "Pipeline for normalizing Kubernetes coredns logs",
+  "processors": [
+    {
+      "script": {
+        "lang": "painless",
+        "source": "ctx.event.created = ctx['@timestamp']; ctx['@timestamp'] = ctx['timestamp']; ctx.remove('timestamp');",
+        "ignore_failure" : true
+      }
+    },
+    {
+      "set": {
+        "field": "source.ip",
+        "value": "{{source.address}}",
+        "if": "ctx.source?.address != null"
+      }
+    },
+    {
+      "convert" : {
+        "field" : "temp.duration",
+        "type": "double"
+      }
+    },
+    {
+      "convert" : {
+        "field" : "coredns.query.size",
+        "type": "long"
+      }
+    },
+    {
+      "convert" : {
+        "field" : "coredns.response.size",
+        "type": "long"
+      }
+    },
+    {
+      "split": {
+        "field": "coredns.response.flags",
+        "separator": "," 
+      }
+    },
+    {
+      "script": {
+        "lang": "painless",
+        "source": "ctx.event.duration = Math.round(ctx.temp.duration * params.scale)",
+        "params": {
+          "scale": 1000000000
+        },
+        "if": "ctx.temp?.duration != null"
+      }
+    },
+    {
+      "remove": {
+        "field": "temp.duration",
+        "ignore_missing": true
+      }
+    }
+  ],
+  "on_failure" : [{ 
+    "set" : { 
+      "field" : "error.message", 
+      "value" : "{{ _ingest.on_failure_message }}" 
+    } 
+  }]
+}

x-pack/filebeat/module/coredns/kubernetes/manifest.yml

@@ -0,0 +1,11 @@
+module_version: 1.0
+
+var:
+  - name: paths
+    default:
+      - /var/lib/docker/containers/*/*-json.log
+  - name: tags
+    default: [coredns]
+
+ingest_pipeline: ingest/pipeline.json
+input: config/coredns.yml

x-pack/filebeat/module/coredns/kubernetes/test/coredns-json.log

@@ -0,0 +1 @@
+{ "message": "2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s", "stream": "stdout", "time": "2019-02-12T00:27:28.903433597Z", "kubernetes": { "container": { "name": "coredns" }, "node": { "name": "minikube" }, "pod": { "uid": "d57d545e-2a9d-11e9-995f-08002730e0dc", "name": "coredns-86c58d9df4-jwhsg" }, "namespace": "kube-system", "replicaset": { "name": "coredns-86c58d9df4" }, "labels": { "pod-template-hash": "86c58d9df4", "k8s-app": "kube-dns" } } }

x-pack/filebeat/module/coredns/kubernetes/test/coredns-json.log-expected.json

@@ -0,0 +1,44 @@
+[
+    {
+        "@timestamp": "2019-02-12T00:27:28.903Z",
+        "coredns.dnssec_ok": "false",
+        "coredns.id": "21583",
+        "coredns.logging_level": "INFO",
+        "coredns.query.class": "IN",
+        "coredns.query.name": "httpbin.org.cluster.local.",
+        "coredns.query.size": 43,
+        "coredns.query.type": "A",
+        "coredns.response.code": "NXDOMAIN",
+        "coredns.response.flags": [
+            "qr",
+            "rd",
+            "ra"
+        ],
+        "coredns.response.size": 136,
+        "ecs.version": "1.0.0-beta2",
+        "event.dataset": "coredns.kubernetes",
+        "event.duration": 102078,
+        "event.module": "coredns",
+        "fileset.name": "kubernetes",
+        "input.type": "log",
+        "kubernetes.container.name": "coredns",
+        "kubernetes.labels.k8s-app": "kube-dns",
+        "kubernetes.labels.pod-template-hash": "86c58d9df4",
+        "kubernetes.namespace": "kube-system",
+        "kubernetes.node.name": "minikube",
+        "kubernetes.pod.name": "coredns-86c58d9df4-jwhsg",
+        "kubernetes.pod.uid": "d57d545e-2a9d-11e9-995f-08002730e0dc",
+        "kubernetes.replicaset.name": "coredns-86c58d9df4",
+        "log.offset": 0,
+        "message": "2019-02-12T00:27:28.903Z [INFO] 172.17.0.4:36413 - 21583 \"A IN httpbin.org.cluster.local. udp 43 false 512\" NXDOMAIN qr,rd,ra 136 0.000102078s",
+        "network.transport": "udp",
+        "service.type": "coredns",
+        "source.address": "172.17.0.4",
+        "source.ip": "172.17.0.4",
+        "source.port": "36413",
+        "stream": "stdout",
+        "tags": [
+            "coredns"
+        ]
+    }
+]

x-pack/filebeat/modules.d/coredns.yml.disabled

@@ -7,7 +7,7 @@
     enabled: true
   # Fileset for Kubernetes deployment
   kubernetes:
-    enabled: false
+    enabled: true
 
     # Set custom paths for the log files. If left empty,
     # Filebeat will choose the paths depending on your OS.