From 6a7a2770e05699be679eb4ce6236de72ed08903d Mon Sep 17 00:00:00 2001 From: Aapo Talvensaari Date: Wed, 6 May 2020 20:39:15 +0300 Subject: [PATCH] feat(conf) add port_map configuration option ### Summary Layer 4 port mapping (or port forwarding) is used a lot when running Kong inside a container. Common example is exposing ports 80 and 443 externally while running Kong inside a container with internal ports 8000 and 8443. It also allows Kong to started without elevated privileges as it is quite common that you need root-access to bind processes to TCP port < 1024. As this translation is done in many cases without using layer 4 proxy_protocol or layer 7 X-Forwarded-Port HTTP header, the Kong has no knowledge about the target port that client originally connected to. This PR adds `port_maps` configuration parameter: ``` #port_maps = # With this configuration parameter you can # let the Kong to know about the port from # which the packets are forwarded to it. This # is fairly common when running Kong in a # containerized or virtualized environment. # For example `port_maps=80:8000, 443:8443` # instructs Kong that the port 80 is mapped # to 8000 (and the port 443 to 8443), where # 8000 and 8443 are the ports the Kong is # listening to. # # This parameter helps Kong to set proper # forwarded upstream HTTP request header or to # get proper forwarded port with a Kong PDK # (in case other means determining it have # failed). It changes routing by a destination # port to route by a port from which packets # are forwarded to Kong, and similarly it # changes the default plugin log serializer to # use the port according to this mapping # instead of reporting the port Kong is # listening to. ``` This gets parsed into `kong.configuration.host_ports` table. E.g. `KONG_PORT_MAP="80:8000,443:8443"` where the first number (`80`/`443`) before `:` is the host published port and the second one (`8000`/`8443`) is the port that Kong is listening translates to following `kong.configuration.host_ports`: ```lua { [8000] = 80, ["8000"] = 80, [8443] = 443, ["8443"] = 443, } ``` This PR also adds a new nginx context variable `ngx.ctx.host_port` which contains this translated port. So instead of using `ngx.var.server_port` the more correct one in many cases is `ngx.ctx.host_port`. Kong PDK is also changed to take this in account when using: ``` local port = kong.request.get_forwarded_port() ``` This commit also changes the `X-Forwarded-Port` to use this. Additionally `stream routing` by `destination port` will use this too. Basic log serializer was changed to use `ngx.var.host_port` as well. Currently it is only usable in proxy/stream, so things like `admin api` is not affected by this. --- kong.conf.default | 23 ++++++ kong/conf_loader.lua | 29 +++++++ kong/pdk/request.lua | 16 +++- kong/plugins/log-serializers/basic.lua | 4 +- kong/router.lua | 3 +- kong/runloop/handler.lua | 14 +++- kong/templates/kong_defaults.lua | 2 + spec/01-unit/03-conf_loader_spec.lua | 29 +++++++ spec/01-unit/10-log_serializer_spec.lua | 8 ++ .../02-integration/05-proxy/01-proxy_spec.lua | 78 +++++++++++++++++++ .../05-proxy/03-upstream_headers_spec.lua | 32 ++++++++ t/01-pdk/04-request/06-get_forwarded_port.t | 55 +++++++++++++ 12 files changed, 289 insertions(+), 4 deletions(-) diff --git a/kong.conf.default b/kong.conf.default index ecc1ac214bfd..0531b12abb96 100644 --- a/kong.conf.default +++ b/kong.conf.default @@ -125,6 +125,29 @@ # This value can be set to `off`, thus disabling # the plugin server and Go plugin loading. +#port_maps = # With this configuration parameter you can + # let the Kong to know about the port from + # which the packets are forwarded to it. This + # is fairly common when running Kong in a + # containerized or virtualized environment. + # For example `port_maps=80:8000, 443:8443` + # instructs Kong that the port 80 is mapped + # to 8000 (and the port 443 to 8443), where + # 8000 and 8443 are the ports the Kong is + # listening to. + # + # This parameter helps Kong to set proper + # forwarded upstream HTTP request header or to + # get proper forwarded port with a Kong PDK + # (in case other means determining it have + # failed). It changes routing by a destination + # port to route by a port from which packets + # are forwarded to Kong, and similarly it + # changes the default plugin log serializer to + # use the port according to this mapping + # instead of reporting the port Kong is + # listening to. + #anonymous_reports = on # Send anonymous usage data such as error # stack traces to help improve Kong. diff --git a/kong/conf_loader.lua b/kong/conf_loader.lua index 480ce6e14e0c..7d7610e48c8c 100644 --- a/kong/conf_loader.lua +++ b/kong/conf_loader.lua @@ -202,6 +202,7 @@ local PREFIX_PATHS = { -- `array`: a comma-separated list local CONF_INFERENCES = { -- forced string inferences (or else are retrieved as numbers) + port_maps = { typ = "array" }, proxy_listen = { typ = "array" }, admin_listen = { typ = "array" }, status_listen = { typ = "array" }, @@ -597,6 +598,34 @@ local function check_and_infer(conf, opts) -- custom validations --------------------- + conf.host_ports = {} + if conf.port_maps then + local MIN_PORT = 1 + local MAX_PORT = 65535 + + for _, port_map in ipairs(conf.port_maps) do + local colpos = string.find(port_map, ":", nil, true) + if not colpos then + errors[#errors + 1] = "invalid port mapping (`port_maps`): " .. port_map + + else + local host_port_str = string.sub(port_map, 1, colpos - 1) + local host_port_num = tonumber(host_port_str, 10) + local kong_port_str = string.sub(port_map, colpos + 1) + local kong_port_num = tonumber(kong_port_str, 10) + + if (host_port_num and host_port_num >= MIN_PORT and host_port_num <= MAX_PORT) + and (kong_port_num and kong_port_num >= MIN_PORT and kong_port_num <= MAX_PORT) + then + conf.host_ports[kong_port_num] = host_port_num + conf.host_ports[kong_port_str] = host_port_num + else + errors[#errors + 1] = "invalid port mapping (`port_maps`): " .. port_map + end + end + end + end + if conf.database == "cassandra" then if string.find(conf.cassandra_lb_policy, "DCAware", nil, true) and not conf.cassandra_local_datacenter diff --git a/kong/pdk/request.lua b/kong/pdk/request.lua index a24e1dbf3f57..c0f3ab3e55ef 100644 --- a/kong/pdk/request.lua +++ b/kong/pdk/request.lua @@ -30,6 +30,7 @@ cjson.decode_array_with_array_mt(true) local function new(self) local _REQUEST = {} + local HOST_PORTS = self.configuration.host_ports or {} local MIN_HEADERS = 1 local MAX_HEADERS_DEFAULT = 100 @@ -200,6 +201,13 @@ local function new(self) -- **Note**: we do not currently offer support for Forwarded HTTP Extension -- (RFC 7239) since it is not supported by ngx_http_realip_module. -- + -- When running Kong behind the L4 port mapping (or forwarding) you can also + -- configure: + -- * [port\_maps](https://getkong.org/docs/latest/configuration/#port_maps) + -- + -- `port_maps` configuration parameter enables this function to return the + -- port to which the port Kong is listening to is mapped to (in case they differ). + -- -- @function kong.request.get_forwarded_port -- @phases rewrite, access, header_filter, body_filter, log, admin_api -- @treturn number the forwarded port @@ -232,7 +240,13 @@ local function new(self) end end - return _REQUEST.get_port() + local host_port = ngx.ctx.host_port + if host_port then + return host_port + end + + local port = _REQUEST.get_port() + return HOST_PORTS[port] or port end diff --git a/kong/plugins/log-serializers/basic.lua b/kong/plugins/log-serializers/basic.lua index a701be0003fa..94a62df91cf8 100644 --- a/kong/plugins/log-serializers/basic.lua +++ b/kong/plugins/log-serializers/basic.lua @@ -51,10 +51,12 @@ function _M.serialize(ngx, kong) end end + local host_port = ctx.host_port or var.server_port + return { request = { uri = request_uri, - url = var.scheme .. "://" .. var.host .. ":" .. var.server_port .. request_uri, + url = var.scheme .. "://" .. var.host .. ":" .. host_port .. request_uri, querystring = kong.request.get_query(), -- parameters, as a table method = kong.request.get_method(), -- http method headers = req_headers, diff --git a/kong/router.lua b/kong/router.lua index aba2e3c4c8b1..575351031152 100644 --- a/kong/router.lua +++ b/kong/router.lua @@ -1755,7 +1755,8 @@ function _M.new(routes) local src_ip = var.remote_addr local src_port = tonumber(var.remote_port, 10) local dst_ip = var.server_addr - local dst_port = tonumber(var.server_port, 10) + local dst_port = tonumber(ngx.ctx.host_port, 10) + or tonumber(var.server_port, 10) -- error value for non-TLS connections ignored intentionally local sni, _ = server_name() diff --git a/kong/runloop/handler.lua b/kong/runloop/handler.lua index e23e4e465ebe..d19db80da3e2 100644 --- a/kong/runloop/handler.lua +++ b/kong/runloop/handler.lua @@ -43,6 +43,9 @@ local COMMA = byte(",") local SPACE = byte(" ") +local HOST_PORTS = {} + + local SUBSYSTEMS = constants.PROTOCOLS_WITH_SUBSYSTEM local EMPTY_T = {} local TTL_ZERO = { ttl = 0 } @@ -871,6 +874,10 @@ return { init_worker = { before = function() + if kong.configuration.host_ports then + HOST_PORTS = kong.configuration.host_ports + end + if kong.configuration.anonymous_reports then reports.configure_ping(kong.configuration) reports.add_ping_value("database_version", kong.db.infos.db_ver) @@ -953,6 +960,8 @@ return { }, preread = { before = function(ctx) + ctx.host_port = HOST_PORTS[var.server_port] or var.server_port + local router = get_updated_router() local match_t = router.exec() @@ -986,6 +995,8 @@ return { }, rewrite = { before = function(ctx) + ctx.host_port = HOST_PORTS[var.server_port] or var.server_port + -- special handling for proxy-authorization and te headers in case -- the plugin(s) want to specify them (store the original) ctx.http_proxy_authorization = var.http_proxy_authorization @@ -1012,7 +1023,8 @@ return { local http_version = ngx.req.http_version() local scheme = var.scheme local host = var.host - local port = tonumber(var.server_port, 10) + local port = tonumber(ctx.host_port, 10) + or tonumber(var.server_port, 10) local content_type = var.content_type local route = match_t.route diff --git a/kong/templates/kong_defaults.lua b/kong/templates/kong_defaults.lua index 2ebefa8a3508..2120a3b1171d 100644 --- a/kong/templates/kong_defaults.lua +++ b/kong/templates/kong_defaults.lua @@ -10,6 +10,8 @@ status_error_log = logs/status_error.log plugins = bundled go_pluginserver_exe = /usr/local/bin/go-pluginserver go_plugins_dir = off +port_maps = NONE +host_ports = NONE anonymous_reports = on proxy_listen = 0.0.0.0:8000 reuseport backlog=16384, 0.0.0.0:8443 http2 ssl reuseport backlog=16384 diff --git a/spec/01-unit/03-conf_loader_spec.lua b/spec/01-unit/03-conf_loader_spec.lua index 4362ed3d6736..cea1bf7a75bb 100644 --- a/spec/01-unit/03-conf_loader_spec.lua +++ b/spec/01-unit/03-conf_loader_spec.lua @@ -445,6 +445,35 @@ describe("Configuration loader", function() end) end) + describe("port_maps and host_ports", function() + it("are empty tables when not specified", function() + local conf = assert(conf_loader(helpers.test_conf_path, {})) + assert.same({}, conf.port_maps) + assert.same({}, conf.host_ports) + end) + it("are tables when specified", function() + local conf = assert(conf_loader(helpers.test_conf_path, { + port_maps = "80:8000,443:8443", + })) + assert.same({ + "80:8000", + "443:8443", + }, conf.port_maps) + assert.same({ + [8000] = 80, + ["8000"] = 80, + [8443] = 443, + ["8443"] = 443, + }, conf.host_ports) + end) + it("gives an error with invalid value", function() + local _, err = conf_loader(helpers.test_conf_path, { + port_maps = "src:dst", + }) + assert.equal("invalid port mapping (`port_maps`): src:dst", err) + end) + end) + describe("inferences", function() it("infer booleans (on/off/true/false strings)", function() local conf = assert(conf_loader()) diff --git a/spec/01-unit/10-log_serializer_spec.lua b/spec/01-unit/10-log_serializer_spec.lua index f37e7fd843e9..81f674066e3a 100644 --- a/spec/01-unit/10-log_serializer_spec.lua +++ b/spec/01-unit/10-log_serializer_spec.lua @@ -79,6 +79,14 @@ describe("Log Serializer", function() assert.is_table(res.tries) end) + it("uses port map (ngx.ctx.host_port) for request url ", function() + ngx.ctx.host_port = 5000 + local res = basic.serialize(ngx, kong) + assert.is_table(res) + assert.is_table(res.request) + assert.equal("http://test.com:5000/request_uri", res.request.url) + end) + it("serializes the matching Route and Services", function() ngx.ctx.route = { id = "my_route" } ngx.ctx.service = { id = "my_service" } diff --git a/spec/02-integration/05-proxy/01-proxy_spec.lua b/spec/02-integration/05-proxy/01-proxy_spec.lua index 594cb4778b18..7e72c9aa779e 100644 --- a/spec/02-integration/05-proxy/01-proxy_spec.lua +++ b/spec/02-integration/05-proxy/01-proxy_spec.lua @@ -129,3 +129,81 @@ describe("#stream proxy interface listeners", function() end end) end) + +for _, strategy in helpers.each_strategy() do + if strategy ~= "off" then + describe("[stream]", function() + local MESSAGE = "echo, ping, pong. echo, ping, pong. echo, ping, pong.\n" + lazy_setup(function() + local bp = helpers.get_db_utils(strategy, { + "routes", + "services", + "plugins", + }) + + local service = assert(bp.services:insert { + host = helpers.mock_upstream_host, + port = helpers.mock_upstream_stream_port, + protocol = "tcp", + }) + + assert(bp.routes:insert { + destinations = { + { port = 19000 }, + }, + protocols = { + "tcp", + }, + service = service, + }) + + assert(helpers.start_kong({ + database = strategy, + stream_listen = helpers.get_proxy_ip(false) .. ":19000, " .. + helpers.get_proxy_ip(false) .. ":18000, " .. + helpers.get_proxy_ip(false) .. ":17000", + port_maps = "19000:18000", + plugins = "bundled,ctx-tests", + nginx_conf = "spec/fixtures/custom_nginx.template", + proxy_listen = "off", + admin_listen = "off", + })) + end) + + lazy_teardown(function() + helpers.stop_kong() + end) + + it("routes by destination port without port map", function() + local tcp_client = ngx.socket.tcp() + assert(tcp_client:connect(helpers.get_proxy_ip(false), 19000)) + assert(tcp_client:send(MESSAGE)) + local body = assert(tcp_client:receive("*a")) + assert.equal(MESSAGE, body) + assert(tcp_client:close()) + end) + + it("uses port maps configuration to route by destination port", function() + local tcp_client = ngx.socket.tcp() + assert(tcp_client:connect(helpers.get_proxy_ip(false), 18000)) + assert(tcp_client:send(MESSAGE)) + local body = assert(tcp_client:receive("*a")) + assert.equal(MESSAGE, body) + assert(tcp_client:close()) + end) + + it("fails to route when no port map is specified and route is not found", function() + local tcp_client = ngx.socket.tcp() + assert(tcp_client:connect(helpers.get_proxy_ip(false), 17000)) + assert(tcp_client:send(MESSAGE)) + local body, err = tcp_client:receive("*a") + if not err then + assert.equal("", body) + else + assert.equal("connection reset by peer", err) + end + assert(tcp_client:close()) + end) + end) + end +end diff --git a/spec/02-integration/05-proxy/03-upstream_headers_spec.lua b/spec/02-integration/05-proxy/03-upstream_headers_spec.lua index fbab37a2a202..6314b02299c1 100644 --- a/spec/02-integration/05-proxy/03-upstream_headers_spec.lua +++ b/spec/02-integration/05-proxy/03-upstream_headers_spec.lua @@ -970,5 +970,37 @@ for _, strategy in helpers.each_strategy() do end) end) end) + + describe("(using port maps configuration)", function() + local proxy_port = helpers.get_proxy_port(false) + + lazy_setup(start_kong { + database = strategy, + nginx_conf = "spec/fixtures/custom_nginx.template", + lua_package_path = "?/init.lua;./kong/?.lua;./spec/fixtures/?.lua", + port_maps = "80:" .. proxy_port, + }) + + lazy_teardown(stop_kong) + + describe("X-Forwarded-Port", function() + it("should be added if not present in request", function() + local headers = request_headers { + ["Host"] = "headers-inspect.com", + } + + assert.equal(80, tonumber(headers["x-forwarded-port"])) + end) + + it("should be replaced if present in request", function() + local headers = request_headers { + ["Host"] = "headers-inspect.com", + ["X-Forwarded-Port"] = "81", + } + + assert.equal(80, tonumber(headers["x-forwarded-port"])) + end) + end) + end) end) end diff --git a/t/01-pdk/04-request/06-get_forwarded_port.t b/t/01-pdk/04-request/06-get_forwarded_port.t index de97f300bd1f..c541f54785a9 100644 --- a/t/01-pdk/04-request/06-get_forwarded_port.t +++ b/t/01-pdk/04-request/06-get_forwarded_port.t @@ -162,3 +162,58 @@ GET /t port: nil --- no_error_log [error] + + + +=== TEST 7: request.get_forwarded_port() returns published port when configured +--- http_config eval: $t::Util::HttpConfig +--- config + location = /t { + access_by_lua_block { + local PDK = require "kong.pdk" + local pdk = PDK.new({ + host_ports = { + [tonumber(ngx.var.server_port)] = 29181 + } + }) + + ngx.say("port: ", pdk.request.get_forwarded_port()) + ngx.say("type: ", type(pdk.request.get_forwarded_port())) + } + } +--- request +GET /t +--- response_body +port: 29181 +type: number +--- no_error_log +[error] + + + +=== TEST 8: request.get_forwarded_port() does not return published port when X-Forwarded-Port is trusted +--- http_config eval: $t::Util::HttpConfig +--- config + location = /t { + access_by_lua_block { + local PDK = require "kong.pdk" + local pdk = PDK.new({ + trusted_ips = { "0.0.0.0/0", "::/0" }, + host_ports = { + [tonumber(ngx.var.server_port)] = 29181 + } + }) + + ngx.say("port: ", pdk.request.get_forwarded_port()) + ngx.say("type: ", type(pdk.request.get_forwarded_port())) + } + } +--- request +GET /t +--- more_headers +X-Forwarded-Port: 1234 +--- response_body +port: 1234 +type: number +--- no_error_log +[error]