[Filebeat] Fix filebeat autodiscover fileset hint for container input (…

…#13296) (#13447) Currently, autodiscover hint assumes that input type is docker. Since docker input is deprecated in 7.2.0, container input should be supported. So we need to set `stream` or `containers.stream` based on input type. Closes #12718 (cherry picked from commit 658772a) Co-authored-by: Kent Wang <pragkent@gmail.com>
elastic · Aug 30, 2019 · 62cc483 · 62cc483
1 parent cd368bd
commit 62cc483
Show file tree

Hide file tree

Showing 3 changed files with 149 additions and 1 deletion.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -136,6 +136,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Syslog input will now omit the `process` object from events if it is empty. {pull}12700[12700]
 - Fix multiline pattern in Postgres which was too permissive {issue}12078[12078] {pull}13069[13069]
 - Allow path variables to be used in files loaded from modules.d. {issue}13184[13184]
+- Fix filebeat autodiscover fileset hint for container input. {pull}13296[13296]
 - Fix incorrect references to index patterns in AWS and CoreDNS dashboards. {pull}13303[13303]
 - Fix timezone parsing of system module ingest pipelines. {pull}13308[13308]
 - Change iis url.path grok pattern from URIPATH to NOTSPACE. {issue}12710[12710] {pull}13225[13225]

diff --git a/filebeat/autodiscover/builder/hints/logs.go b/filebeat/autodiscover/builder/hints/logs.go
@@ -22,6 +22,7 @@ import (
 	"regexp"
 
 	"github.com/elastic/beats/filebeat/fileset"
+	"github.com/elastic/beats/filebeat/harvester"
 	"github.com/elastic/beats/libbeat/autodiscover"
 	"github.com/elastic/beats/libbeat/autodiscover/builder"
 	"github.com/elastic/beats/libbeat/autodiscover/template"
@@ -140,7 +141,12 @@ func (l *logHints) CreateConfig(event bus.Event) []*common.Config {
 		filesets := l.getFilesets(hints, module)
 		for fileset, conf := range filesets {
 			filesetConf, _ := common.NewConfigFrom(config)
-			filesetConf.SetString("containers.stream", -1, conf.Stream)
+
+			if inputType, _ := filesetConf.String("type", -1); inputType == harvester.ContainerType {
+				filesetConf.SetString("stream", -1, conf.Stream)
+			} else {
+				filesetConf.SetString("containers.stream", -1, conf.Stream)
+			}
 
 			moduleConf[fileset+".enabled"] = conf.Enabled
 			moduleConf[fileset+".input"] = filesetConf

diff --git a/filebeat/autodiscover/builder/hints/logs_test.go b/filebeat/autodiscover/builder/hints/logs_test.go
@@ -473,6 +473,147 @@ func TestGenerateHints(t *testing.T) {
 				},
 			},
 		},
+		{
+			msg:    "Hint with module should attach input to its filesets",
+			config: defaultCfg,
+			event: bus.Event{
+				"host": "1.2.3.4",
+				"kubernetes": common.MapStr{
+					"container": common.MapStr{
+						"name": "foobar",
+						"id":   "abc",
+					},
+				},
+				"container": common.MapStr{
+					"name": "foobar",
+					"id":   "abc",
+				},
+				"hints": common.MapStr{
+					"logs": common.MapStr{
+						"module": "apache2",
+					},
+				},
+			},
+			len: 1,
+			result: common.MapStr{
+				"module": "apache2",
+				"error": map[string]interface{}{
+					"enabled": true,
+					"input": map[string]interface{}{
+						"type":   "container",
+						"stream": "all",
+						"paths": []interface{}{
+							"/var/lib/docker/containers/abc/*-json.log",
+						},
+					},
+				},
+				"access": map[string]interface{}{
+					"enabled": true,
+					"input": map[string]interface{}{
+						"type":   "container",
+						"stream": "all",
+						"paths": []interface{}{
+							"/var/lib/docker/containers/abc/*-json.log",
+						},
+					},
+				},
+			},
+		},
+		{
+			msg:    "Hint with module should honor defined filesets",
+			config: defaultCfg,
+			event: bus.Event{
+				"host": "1.2.3.4",
+				"kubernetes": common.MapStr{
+					"container": common.MapStr{
+						"name": "foobar",
+						"id":   "abc",
+					},
+				},
+				"container": common.MapStr{
+					"name": "foobar",
+					"id":   "abc",
+				},
+				"hints": common.MapStr{
+					"logs": common.MapStr{
+						"module":  "apache2",
+						"fileset": "access",
+					},
+				},
+			},
+			len: 1,
+			result: common.MapStr{
+				"module": "apache2",
+				"access": map[string]interface{}{
+					"enabled": true,
+					"input": map[string]interface{}{
+						"type":   "container",
+						"stream": "all",
+						"paths": []interface{}{
+							"/var/lib/docker/containers/abc/*-json.log",
+						},
+					},
+				},
+				"error": map[string]interface{}{
+					"enabled": false,
+					"input": map[string]interface{}{
+						"type":   "container",
+						"stream": "all",
+						"paths": []interface{}{
+							"/var/lib/docker/containers/abc/*-json.log",
+						},
+					},
+				},
+			},
+		},
+		{
+			msg:    "Hint with module should honor defined filesets with streams",
+			config: defaultCfg,
+			event: bus.Event{
+				"host": "1.2.3.4",
+				"kubernetes": common.MapStr{
+					"container": common.MapStr{
+						"name": "foobar",
+						"id":   "abc",
+					},
+				},
+				"container": common.MapStr{
+					"name": "foobar",
+					"id":   "abc",
+				},
+				"hints": common.MapStr{
+					"logs": common.MapStr{
+						"module":         "apache2",
+						"fileset.stdout": "access",
+						"fileset.stderr": "error",
+					},
+				},
+			},
+			len: 1,
+			result: common.MapStr{
+				"module": "apache2",
+				"access": map[string]interface{}{
+					"enabled": true,
+					"input": map[string]interface{}{
+						"type":   "container",
+						"stream": "stdout",
+						"paths": []interface{}{
+							"/var/lib/docker/containers/abc/*-json.log",
+						},
+					},
+				},
+				"error": map[string]interface{}{
+					"enabled": true,
+					"input": map[string]interface{}{
+						"type":   "container",
+						"stream": "stderr",
+						"paths": []interface{}{
+							"/var/lib/docker/containers/abc/*-json.log",
+						},
+					},
+				},
+			},
+		},
 	}
 
 	for _, test := range tests {