Secure API

config/logstash.yml

@@ -118,26 +118,54 @@
 #
 # config.support_escapes: false
 #
-# ------------ HTTP API Settings -------------
+# ------------ API Settings -------------
 # Define settings related to the HTTP API here.
 #
 # The HTTP API is enabled by default. It can be disabled, but features that rely
 # on it will not work as intended.
-# http.enabled: true
 #
-# By default, the HTTP API is bound to only the host's local loopback interface,
-# ensuring that it is not accessible to the rest of the network. Because the API
-# includes neither authentication nor authorization and has not been hardened or
-# tested for use as a publicly-reachable API, binding to publicly accessible IPs
-# should be avoided where possible.
+# api.enabled: true
 #
-# http.host: 127.0.0.1
+# By default, the HTTP API is not secured and is therefore bound to only the
+# host's loopback interface, ensuring that it is not accessible to the rest of
+# the network.
+# When secured with SSL and Basic Auth, the API is bound to _all_ interfaces
+# unless configured otherwise.
+#
+# api.http.host: 127.0.0.1
 #
 # The HTTP API web server will listen on an available port from the given range.
 # Values can be specified as a single port (e.g., `9600`), or an inclusive range
 # of ports (e.g., `9600-9700`).
 #
-# http.port: 9600-9700
+# api.http.port: 9600-9700
+#
+# The HTTP API includes a customizable "environment" value in its response,
+# which can be configured here.
+#
+# api.environment: "production"
+#
+# The HTTP API can be secured with SSL (TLS). To do so, you will need to provide
+# the path to a password-protected keystore in p12 or jks format, along with credentials.
+#
+# api.ssl.enabled: false
+# api.ssl.keystore.path: /path/to/keystore.jks
+# api.ssl.keystore.password: "y0uRp4$$w0rD"
+#
+# The HTTP API can be configured to require authentication. Acceptable values are
+#  - `none`:  no auth is required (default)
+#  - `basic`: clients must authenticate with HTTP Basic auth, as configured
+#             with `api.auth.basic.*` options below
+# api.auth.type: none
+#
+# When configured with `api.auth.type` `basic`, you must provide the credentials
+# that requests will be validated against. Usage of Environment or Keystore
+# variable replacements is encouraged (such as the value `"${HTTP_PASS}"`, which
+# resolves to the value stored in the keystore's `HTTP_PASS` variable if present
+# or the same variable from the environment)
+#
+# api.auth.basic.username: "logstash-user"
+# api.auth.basic.password: "s3cUreP4$$w0rD"
 #
 # ------------ Module Settings ---------------
 # Define modules here.  Modules definitions must be defined as an array.
@@ -240,17 +268,6 @@
 #
 # path.dead_letter_queue:
 #
-# ------------ Metrics Settings --------------
-#
-# Bind address for the metrics REST endpoint
-#
-# http.host: "127.0.0.1"
-#
-# Bind port for the metrics REST endpoint, this option also accept a range
-# (9600-9700) and logstash will pick up the first available ports.
-#
-# http.port: 9600-9700
-#
 # ------------ Debugging Settings --------------
 #
 # Options for log.level:

docker/data/logstash/env2yaml/env2yaml.go

@@ -48,6 +48,10 @@ func squashSetting(setting string) string {
 // return the canonical setting name. eg. 'pipeline.unsafe_shutdown'
 func normalizeSetting(setting string) (string, error) {
 	valid_settings := []string{
+		"api.enabled",
+		"api.http.host",
+		"api.http.port",
+		"api.environment",
 		"node.name",
 		"path.data",
 		"pipeline.id",
@@ -80,9 +84,10 @@ func normalizeSetting(setting string) (string, error) {
 		"dead_letter_queue.max_bytes",
 		"dead_letter_queue.flush_interval",
 		"path.dead_letter_queue",
-		"http.host",
-		"http.port",
-		"http.enabled",
+		"http.enabled",     // DEPRECATED: prefer `api.enabled`
+		"http.environment", // DEPRECATED: prefer `api.environment`
+		"http.host",        // DEPRECATED: prefer `api.http.host`
+		"http.port",        // DEPRECATED: prefer `api.http.port`
 		"log.level",
 		"log.format",
 		"modules",

docs/static/logging.asciidoc

@@ -55,7 +55,7 @@ downtime. Instead, you can dynamically update logging levels through the logging
 immediately and do not need a restart. 
 
 NOTE: By default, the logging API attempts to bind to `tcp:9600`. If this port is already in use by another Logstash
-instance, you need to launch Logstash with the `--http.port` flag specified to bind to a different port. See
+instance, you need to launch Logstash with the `--api.http.port` flag specified to bind to a different port. See
 <<command-line-flags>> for more information.
 
 ===== Retrieve list of logging configurations

docs/static/monitoring/monitoring-apis.asciidoc

@@ -31,9 +31,39 @@ Example response:
 --------------------------------------------------
 
 NOTE: By default, the monitoring API attempts to bind to `tcp:9600`. If this port is already in use by another Logstash
-instance, you need to launch Logstash with the `--http.port` flag specified to bind to a different port. See
+instance, you need to launch Logstash with the `--api.http.port` flag specified to bind to a different port. See
 <<command-line-flags>> for more information.
 
+[discrete]
+[[monitoring-api-security]]
+==== Securing the Logstash API
+
+The {ls} Monitoring APIs are not secured by default, but you can configure {ls} to secure them in one of several ways to meet your organization's needs.
+
+You can enable SSL for the Logstash API by setting `api.ssl.enabled: true` in the `logstash.yml`, and providing the relevant keystore settings `api.ssl.keystore.path` and `api.ssl.keystore.password`:
+
+[source]
+--------------------------------------------------
+api.ssl.enabled: true
+api.ssl.keystore.path: /path/to/keystore.jks
+api.ssl.keystore.password: "s3cUr3p4$$w0rd"
+--------------------------------------------------
+
+The keystore must be in either jks or p12 format, and must contain both a certificate and a private key.
+Connecting clients receive this certificate, allowing them to authenticate the Logstash endpoint.
+
+You can also require HTTP Basic authentication by setting `api.auth.type: basic` in the `logstash.yml`, and providing the relevant credentials `api.auth.basic.username` and `api.auth.basic.password`:
+
+[source]
+--------------------------------------------------
+api.auth.type: basic
+api.auth.basic.username: "logstash"
+api.auth.basic.password: "stashy"
+--------------------------------------------------
+
+NOTE: Usage of Keystore or Environment or variable replacements is encouraged for password-type fields to avoid storing them in plain text.
+      For example, specifying the value `"${HTTP_PASS}"` will resolve to the value stored in the <<keystore,secure keystore's>> `HTTP_PASS` variable if present or the same variable from the <<environment-variables,environment>>)
+
 [discrete]
 [[monitoring-common-options]]
 ==== Common options

docs/static/running-logstash-command-line.asciidoc

@@ -207,10 +207,13 @@ With this command, Logstash concatenates three config files, `/tmp/one`, `/tmp/t
   How frequently to poll the configuration location for changes. The default value is "3s".
   Note that the unit qualifier (`s`) is required.
 
-*`--http.host HTTP_HOST`*::
+*`--api-enabled ENABLED`*::
+  The HTTP API is enabled by default, but can be disabled by passing `false` to this option.
+
+*`--api.http.host HTTP_HOST`*::
   Web API binding host. This option specifies the bind address for the metrics REST endpoint. The default is "127.0.0.1".
 
-*`--http.port HTTP_PORT`*::
+*`--api.http.port HTTP_PORT`*::
   Web API http port. This option specifies the bind port for the metrics REST endpoint. The default is 9600-9700.
   This setting accepts a range of the format 9600-9700. Logstash will pick up the first available port.
 

docs/static/settings-file.asciidoc

@@ -235,13 +235,53 @@ Values other than `disabled` are currently considered BETA, and may produce unin
 | The directory path where the data files will be stored for the dead-letter queue.
 | `path.data/dead_letter_queue`
 
-| `http.host`
-| The bind address for the metrics REST endpoint.
+| `api.enabled`
+|  The HTTP API is enabled by default. It can be disabled, but features that rely on it will not work as intended.
+| `true`
+
+| `api.environment`
+| The API returns the provided string as a part of its response. Setting your environment may help to disambiguate between similarly-named nodes in production vs test environments.
+| `production`
+
+| `api.http.host`
+| The bind address for the HTTP API endpoint.
+  By default, the {ls} HTTP API binds only to the local loopback interface.
+  When configured securely (`api.ssl.enabled: true` and `api.auth.type: basic`), the HTTP API binds to _all_ available interfaces.
 | `"127.0.0.1"`
 
-| `http.port`
-| The bind port for the metrics REST endpoint.
-| `9600`
+| `api.http.port`
+| The bind port for the HTTP API endpoint.
+| `9600-9700`
+
+| `api.ssl.enabled`
+| Set to `true` to enable SSL on the HTTP API.
+  Doing so requires both `api.ssl.keystore.path` and `api.ssl.keystore.password` to be set.
+| `false`
+
+| `api.ssl.keystore.path`
+| The path to a valid JKS or PKCS12 keystore for use in securing the {ls} API.
+  The keystore must be password-protected, and must contain a single certificate chain and a private key.
+  This setting is ignored unless `api.ssl.enabled` is set to `true`.
+| _N/A_
+
+| `api.ssl.keystore.password`
+| The password to the keystore provided with `api.ssl.keystore.path`.
+  This setting is ignored unless `api.ssl.enabled` is set to `true`.
+| _N/A_
+
+| `api.auth.type`
+| Set to `basic` to require HTTP Basic auth on the API using the credentials supplied with `api.auth.basic.username` and `api.auth.basic.password`.
+| `none`
+
+| `api.auth.basic.username`
+| The username to require for HTTP Basic auth
+  Ignored unless `api.auth.type` is set to `basic`.
+| _N/A_
+
+| `api.auth.basic.password`
+| The password to require for HTTP Basic auth
+Ignored unless `api.auth.type` is set to `basic`.
+| _N/A_
 
 | `log.level`
 a|

logstash-core/lib/logstash/agent.rb

@@ -62,9 +62,6 @@ def initialize(settings = LogStash::SETTINGS, source_loader = nil)
     @pipelines_registry = LogStash::PipelinesRegistry.new
 
     @name = setting("node.name")
-    @http_host = setting("http.host")
-    @http_port = setting("http.port")
-    @http_environment = setting("http.environment")
     # Generate / load the persistent uuid
     id
 
@@ -76,6 +73,9 @@ def initialize(settings = LogStash::SETTINGS, source_loader = nil)
       end
     end
 
+    # Initialize, but do not start the webserver.
+    @webserver = LogStash::WebServer.from_settings(@logger, self, settings)
+
     # This is for backward compatibility in the tests
     if source_loader.nil?
       @source_loader = LogStash::Config::SourceLoader.new
@@ -427,17 +427,15 @@ def dispatch_events(converge_results)
   end
 
   def start_webserver_if_enabled
-    if @settings.get_value("http.enabled")
+    if @settings.get_value("api.enabled")
       start_webserver
     else
-      @logger.info("HTTP API is disabled (`http.enabled=false`); webserver will not be started.")
+      @logger.info("HTTP API is disabled (`api.enabled=false`); webserver will not be started.")
     end
   end
 
   def start_webserver
     @webserver_control_lock.synchronize do
-      options = {:http_host => @http_host, :http_ports => @http_port, :http_environment => @http_environment }
-      @webserver = LogStash::WebServer.new(@logger, self, options)
       @webserver_thread = Thread.new(@webserver) do |webserver|
         LogStash::Util.set_thread_name("Api Webserver")
         webserver.run
@@ -447,7 +445,7 @@ def start_webserver
 
   def stop_webserver
     @webserver_control_lock.synchronize do
-      if @webserver
+      if @webserver_thread
         @webserver.stop
         if @webserver_thread.join(5).nil?
           @webserver_thread.kill

logstash-core/lib/logstash/environment.rb

@@ -69,10 +69,16 @@ module Environment
            Setting::Boolean.new("help", false),
             Setting::Boolean.new("enable-local-plugin-development", false),
             Setting::String.new("log.format", "plain", true, ["json", "plain"]),
-           Setting::Boolean.new("http.enabled", true),
-            Setting::String.new("http.host", "127.0.0.1"),
-         Setting::PortRange.new("http.port", 9600..9700),
-            Setting::String.new("http.environment", "production"),
+           Setting::Boolean.new("api.enabled", true).with_deprecated_alias("http.enabled"),
+            Setting::String.new("api.http.host", "127.0.0.1").with_deprecated_alias("http.host"),
+         Setting::PortRange.new("api.http.port", 9600..9700).with_deprecated_alias("http.port"),
+            Setting::String.new("api.environment", "production").with_deprecated_alias("http.environment"),
+            Setting::String.new("api.auth.type", "none", true, %w(none basic)),
+            Setting::String.new("api.auth.basic.username", nil, false).nullable,
+          Setting::Password.new("api.auth.basic.password", nil, false).nullable,
+           Setting::Boolean.new("api.ssl.enabled", false),
+  Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable,
+          Setting::Password.new("api.ssl.keystore.password", nil, false).nullable,
             Setting::String.new("queue.type", "memory", true, ["persisted", "memory"]),
             Setting::Boolean.new("queue.drain", false),
             Setting::Bytes.new("queue.page_capacity", "64mb"),

logstash-core/lib/logstash/patches/clamp.rb

@@ -78,8 +78,17 @@ def define_deprecated_accessors_for(option, opts, &block)
 
       def define_deprecated_writer_for(option, opts, &block)
         define_method(option.write_method) do |value|
-          LogStash::DeprecationMessage.instance << "DEPRECATION WARNING: The flag #{option.switches} has been deprecated, please use \"--#{opts[:new_flag]}=#{opts[:new_value]}\" instead."
-          LogStash::SETTINGS.set(opts[:new_flag], opts[:new_value])
+          new_flag = opts[:new_flag]
+          new_value = opts.fetch(:new_value, value)
+          passthrough = opts.fetch(:passthrough, false)
+
+          LogStash::DeprecationMessage.instance << "DEPRECATION WARNING: The flag #{option.switches} has been deprecated, please use \"--#{new_flag}=#{new_value}\" instead."
+
+          if passthrough
+            LogStash::SETTINGS.set(option.attribute_name, value)
+          else
+            LogStash::SETTINGS.set(opts[:new_flag], opts.include?(:new_value) ? opts[:new_value] : value)
+          end
         end
       end
     end

logstash-core/lib/logstash/patches/puma.rb

@@ -36,6 +36,14 @@ def sync=(v)
       # noop
     end
 
+    def sync
+      # noop
+    end
+
+    def flush
+      # noop
+    end
+
     def logger=(logger)
       @logger_lock.synchronize { @logger = logger }
     end
@@ -48,6 +56,14 @@ def puts(str)
     alias_method :<<, :puts
   end
 
+  # ::Puma::Events#error(str) sends Kernel#exit
+  # let's raise something sensible instead.
+  UnrecoverablePumaError = Class.new(RuntimeError)
+  class NonCrashingPumaEvents < ::Puma::Events
+    def error(str)
+      raise UnrecoverablePumaError.new(str)
+    end
+  end
 end
 
 # Reopen the puma class to create a scoped STDERR and STDOUT