Fix panic with Logstash output due to no type (#4331)

* Fix panic due to no type Some Beats no longer add a "type", but the LS output was still requiring it. This hardcodes the type in `@metadata` to `doc`, to keep BWC with older Logstashes/configs. It also removes `"type"` from a few kafka integration tests, to make sure we don't depend on it. * Added breaking + deprecation changelog and docs updates
elastic · May 17, 2017 · 97de042 · 97de042
1 parent 03690a8
commit 97de042
Show file tree

Hide file tree

Showing 6 changed files with 14 additions and 17 deletions.
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -16,6 +16,7 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 
 *Filebeat*
 - Rename `input_type` field to `prospector.type` {pull}4294[4294]
+- The `@metadata.type` field, added by the Logstash output, is now hardcoded to `doc` and will be removed in future versions. {pull}4331[4331].
 
 *Heartbeat*
 
@@ -73,6 +74,8 @@ https://github.com/elastic/beats/compare/v6.0.0-alpha1...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- The `@metadata.type` field, added by the Logstash output, is deprecated, hardcoded to `doc` and will be removed in future versions. {pull}4331[4331].
+
 *Filebeat*
 - Deprecate `input_type` prospector config. Use `type` config option instead. {pull}4294[4294]
 

diff --git a/libbeat/docs/outputconfig.asciidoc b/libbeat/docs/outputconfig.asciidoc
@@ -318,7 +318,7 @@ use in Logstash for indexing and filtering:
     ...
     "@metadata": { <1>
       "beat": "{beatname_lc}", <2>
-      "type": "<event type>" <3>
+      "type": "doc" <3>
     }
 }
 ------------------------------------------------------------------------------
@@ -329,7 +329,12 @@ events sent from Logstash. See the
 for more about the `@metadata` field.
 <2> The default is {beatname_lc}. To change this value, set the
 <<logstash-index,`index`>> option in the {beatname_uc} config file.
-<3> The value of `type` varies depending on the event type.
+<3> The value of `type` is currently hardcoded to `doc`. It was used by previous
+Logstash configs to set the type of the document in Elasticsearch.
+
+
+WARNING: The `@metadata.type` field, added by the Logstash output, is
+deprecated, hardcoded to `doc`, and will be removed in {beatname_uc} 7.0.
 
 You can access this metadata from within the Logstash config file to set values
 dynamically based on the contents of the metadata.
@@ -351,20 +356,18 @@ output {
   elasticsearch {
     hosts => ["http://localhost:9200"]
     index => "%{[@metadata][beat]}-%{+YYYY.MM.dd}" <1>
-    document_type => "%{[@metadata][type]}" <2>
   }
 }
 ------------------------------------------------------------------------------
 <1> `%{[@metadata][beat]}` sets the first part of the index name to the value
 of the `beat` metadata field, and `%{+YYYY.MM.dd}` sets the second part of the
 name to a date based on the Logstash `@timestamp` field. For example:
 +{beatname_lc}-2017.03.29+.
-<2> `%{[@metadata][type]}` sets the document type based on the value of the `type`
-metadata field.
 
 Events indexed into Elasticsearch with the Logstash configuration shown here
 will be similar to events directly indexed by Beats into Elasticsearch.
 
+
 ==== Compatibility
 
 This output works with all compatible versions of Logstash. See "Supported Beats Versions" in the https://www.elastic.co/support/matrix#show_compatibility[Elastic Support Matrix].

diff --git a/libbeat/outputs/kafka/kafka_integration_test.go b/libbeat/outputs/kafka/kafka_integration_test.go
@@ -54,7 +54,6 @@ func TestKafkaPublish(t *testing.T) {
 			single(common.MapStr{
 				"@timestamp": common.Time(time.Now()),
 				"host":       "test-host",
-				"type":       "log",
 				"message":    id,
 			}),
 		},
@@ -80,7 +79,6 @@ func TestKafkaPublish(t *testing.T) {
 			single(common.MapStr{
 				"@timestamp": common.Time(time.Now()),
 				"host":       "test-host",
-				"type":       "log",
 				"message":    id,
 			}),
 		},
@@ -91,7 +89,6 @@ func TestKafkaPublish(t *testing.T) {
 			randMulti(5, 100, common.MapStr{
 				"@timestamp": common.Time(time.Now()),
 				"host":       "test-host",
-				"type":       "log",
 			}),
 		},
 		{

diff --git a/libbeat/outputs/kafka/partition_test.go b/libbeat/outputs/kafka/partition_test.go
@@ -210,7 +210,6 @@ func partTestSimple(N int, makeKey bool) partTestScenario {
 
 			event := common.MapStr{
 				"@timestamp": common.Time(ts),
-				"type":       "test",
 				"message":    randString(20),
 			}
 
@@ -263,7 +262,6 @@ func partTestHashInvariant(N int) partTestScenario {
 
 			event := common.MapStr{
 				"@timestamp": common.Time(ts),
-				"type":       "test",
 				"message":    randString(20),
 			}
 

diff --git a/libbeat/outputs/logstash/json.go b/libbeat/outputs/logstash/json.go
@@ -34,14 +34,11 @@ func makeLogstashEventEncoder(beat string) (func(interface{}) ([]byte, error), e
 
 		buf.WriteRune('{')
 		if _, hasMeta := event["@metadata"]; !hasMeta {
-			typ := event["type"].(string)
-			buf.WriteString(`"@metadata":{"type":`)
-			encodeString(buf, typ)
+			buf.WriteString(`"@metadata":{"type":"doc","beat":`)
+			buf.Write(beatName)
+			buf.WriteString(`},`)
 		}
 
-		buf.WriteString(`,"beat":`)
-		buf.Write(beatName)
-		buf.WriteString(`},`)
 		err := enc.encodeKeyValues(event)
 		if err != nil {
 			logp.Err("jsonEncode failed with: %v", err)

diff --git a/libbeat/outputs/logstash/logstash_integration_test.go b/libbeat/outputs/logstash/logstash_integration_test.go
@@ -266,7 +266,6 @@ func testSendMessageViaLogstash(t *testing.T, name string, tls bool) {
 	event := outputs.Data{Event: common.MapStr{
 		"@timestamp": common.Time(time.Now()),
 		"host":       "test-host",
-		"type":       "log",
 		"message":    "hello world",
 	}}
 	ls.PublishEvent(nil, testOptions, event)