[Filebeat][http_endpoint input] Adds support for custom auth header names and secret

CHANGELOG.next.asciidoc

@@ -505,6 +505,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add event.ingested for CrowdStrike module {pull}20138[20138]
 - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138]
 - Add event.ingested for Suricata module {pull}20220[20220]
+- Add support for custom header and headersecret for filebeat http_endpoint input {pull}20435[20435]
 - Add event.ingested to all Filebeat modules. {pull}20386[20386]
 
 *Heartbeat*

x-pack/filebeat/docs/inputs/input-http-endpoint.asciidoc

@@ -70,6 +70,18 @@ Basic auth and SSL example:
   password: somepassword
 ----
 
+Authentication or checking that a specific header includes a specific value
+["source","yaml",subs="attributes"]
+----
+{beatname_lc}.inputs:
+- type: http_endpoint
+  enabled: true
+  listen_address: 192.168.1.1
+  listen_port: 8080
+  secret.header: someheadername
+  secret.value: secretheadertoken
+----
+
 
 ==== Configuration options
 
@@ -91,6 +103,16 @@ If `basic_auth` is enabled, this is the username used for authentication against
 
 If `basic_auth` is eanbled, this is the password used for authentication against the HTTP listener. Requires `username` to also be set.
 
+[float]
+==== `secret.header`
+
+The header to check for a specific value specified by `secret.value`. Certain webhooks provide the possibility to include a special header and secret to identify the source.
+
+[float]
+==== `secret.value`
+
+The secret stored in the header name specified by `secret.header`. Certain webhooks provide the possibility to include a special header and secret to identify the source.
+
 [float]
 ==== `content_type`
 

x-pack/filebeat/input/http_endpoint/config.go

@@ -24,6 +24,8 @@ type config struct {
 	URL           string                  `config:"url"`
 	Prefix        string                  `config:"prefix"`
 	ContentType   string                  `config:"content_type"`
+	SecretHeader  string                  `config:"secret.header"`
+	SecretValue   string                  `config:"secret.value"`
 }
 
 func defaultConfig() config {
@@ -38,6 +40,8 @@ func defaultConfig() config {
 		URL:           "/",
 		Prefix:        "json",
 		ContentType:   "application/json",
+		SecretHeader:  "",
+		SecretValue:   "",
 	}
 }
 
@@ -52,5 +56,9 @@ func (c *config) Validate() error {
 		}
 	}
 
+	if (c.SecretHeader != "" && c.SecretValue == "") || (c.SecretHeader == "" && c.SecretValue != "") {
+		return errors.New("Both secret.header and secret.value must be set")
+	}
+
 	return nil
 }

x-pack/filebeat/input/http_endpoint/input.go

@@ -83,11 +83,13 @@ func (e *httpEndpoint) Run(ctx v2.Context, publisher stateless.Publisher) error
 	log := ctx.Logger.With("address", e.addr)
 
 	validator := &apiValidator{
-		basicAuth:   e.config.BasicAuth,
-		username:    e.config.Username,
-		password:    e.config.Password,
-		method:      http.MethodPost,
-		contentType: e.config.ContentType,
+		basicAuth:    e.config.BasicAuth,
+		username:     e.config.Username,
+		password:     e.config.Password,
+		method:       http.MethodPost,
+		contentType:  e.config.ContentType,
+		secretHeader: e.config.SecretHeader,
+		secretValue:  e.config.SecretValue,
 	}
 
 	handler := &httpHandler{

x-pack/filebeat/input/http_endpoint/validate.go

@@ -21,9 +21,12 @@ type apiValidator struct {
 	username, password string
 	method             string
 	contentType        string
+	secretHeader       string
+	secretValue        string
 }
 
 var errIncorrectUserOrPass = errors.New("Incorrect username or password")
+var errIncorrectHeaderSecret = errors.New("Incorrect header or header secret")
 
 func (v *apiValidator) ValidateHeader(r *http.Request) (int, error) {
 	if v.basicAuth {
@@ -33,6 +36,12 @@ func (v *apiValidator) ValidateHeader(r *http.Request) (int, error) {
 		}
 	}
 
+	if v.secretHeader != "" && v.secretValue != "" {
+		if v.secretValue != r.Header.Get(v.secretHeader) {
+			return http.StatusUnauthorized, errIncorrectHeaderSecret
+		}
+	}
+
 	if v.method != "" && v.method != r.Method {
 		return http.StatusMethodNotAllowed, fmt.Errorf("Only %v requests supported", v.method)
 	}

x-pack/filebeat/tests/system/test_http_endpoint.py

@@ -79,6 +79,8 @@ def test_http_endpoint_request(self):
 
         output = self.read_output()
 
+        print("response:", r.status_code, r.text)
+
         assert r.text == '{"message": "success"}'
         assert output[0]["input.type"] == "http_endpoint"
         assert output[0]["json.{}".format(self.prefix)] == message
@@ -98,6 +100,8 @@ def test_http_endpoint_wrong_content_header(self):
 
         filebeat.check_kill_and_wait()
 
+        print("response:", r.status_code, r.text)
+
         assert r.status_code == 415
         assert r.text == '{"message": "Wrong Content-Type header, expecting application/json"}'
 
@@ -135,9 +139,59 @@ def test_http_endpoint_wrong_auth_value(self):
 
         filebeat.check_kill_and_wait()
 
+        print("response:", r.status_code, r.text)
+
         assert r.status_code == 401
         assert r.text == '{"message": "Incorrect username or password"}'
 
+    def test_http_endpoint_wrong_auth_header(self):
+        """
+        Test http_endpoint input with wrong auth header and secret.
+        """
+        options = """
+  secret.header: Authorization
+  secret.value: 123password
+"""
+        self.get_config(options)
+        filebeat = self.start_beat()
+        self.wait_until(lambda: self.log_contains("Starting HTTP server on {}:{}".format(self.host, self.port)))
+
+        message = "somerandommessage"
+        payload = {self.prefix: message}
+        headers = {"Content-Type": "application/json", "Authorization": "password123"}
+        r = requests.post(self.url, headers=headers, data=json.dumps(payload))
+
+        filebeat.check_kill_and_wait()
+
+        print("response:", r.status_code, r.text)
+
+        assert r.status_code == 401
+        assert r.text == '{"message": "Incorrect header or header secret"}'
+
+    def test_http_endpoint_correct_auth_header(self):
+        """
+        Test http_endpoint input with correct auth header and secret.
+        """
+        options = """
+  secret.header: Authorization
+  secret.value: 123password
+"""
+        self.get_config(options)
+        filebeat = self.start_beat()
+        self.wait_until(lambda: self.log_contains("Starting HTTP server on {}:{}".format(self.host, self.port)))
+
+        message = "somerandommessage"
+        payload = {self.prefix: message}
+        headers = {"Content-Type": "application/json", "Authorization": "123password"}
+        r = requests.post(self.url, headers=headers, data=json.dumps(payload))
+
+        filebeat.check_kill_and_wait()
+        output = self.read_output()
+
+        assert r.text == '{"message": "success"}'
+        assert output[0]["input.type"] == "http_endpoint"
+        assert output[0]["json.{}".format(self.prefix)] == message
+
     def test_http_endpoint_empty_body(self):
         """
         Test http_endpoint input with empty body.
@@ -151,6 +205,8 @@ def test_http_endpoint_empty_body(self):
 
         filebeat.check_kill_and_wait()
 
+        print("response:", r.status_code, r.text)
+
         assert r.status_code == 406
         assert r.text == '{"message": "Body cannot be empty"}'
 
@@ -169,6 +225,7 @@ def test_http_endpoint_malformed_json(self):
         filebeat.check_kill_and_wait()
 
         print("response:", r.status_code, r.text)
+
         assert r.status_code == 400
         assert r.text.startswith('{"message": "Malformed JSON body:')
 
@@ -184,8 +241,9 @@ def test_http_endpoint_get_request(self):
         payload = {self.prefix: message}
         headers = {"Content-Type": "application/json", "Accept": "application/json"}
         r = requests.get(self.url, headers=headers, data=json.dumps(payload))
-        print("response:", r.status_code, r.text)
         filebeat.check_kill_and_wait()
 
+        print("response:", r.status_code, r.text)
+
         assert r.status_code == 405
         assert r.text == '{"message": "Only POST requests supported"}'