fix(core): relocate unix sockets to a subdirectory

build/dockerfiles/entrypoint.sh

@@ -46,11 +46,12 @@ if [[ "$1" == "kong" ]]; then
 
     # remove all dangling sockets in $PREFIX dir before starting Kong
     LOGGED_SOCKET_WARNING=0
-    for localfile in "$PREFIX"/*; do
+    runtime_prefix=$PREFIX/runtime
+    for localfile in "$runtime_prefix"/*; do
       if [ -S "$localfile" ]; then
         if (( LOGGED_SOCKET_WARNING == 0 )); then
-          printf >&2 'WARN: found dangling unix sockets in the prefix directory '
-          printf >&2 '(%q) ' "$PREFIX"
+          printf >&2 'WARN: found dangling unix sockets in the runtime prefix '
+          printf >&2 '(%q) ' "$runtime_prefix"
           printf >&2 'while preparing to start Kong. This may be a sign that Kong '
           printf >&2 'was previously shut down uncleanly or is in an unknown state '
           printf >&2 'and could require further investigation.\n'

changelog/unreleased/kong/move-sockets-to-subdir.yml

@@ -0,0 +1,3 @@
+message: Moved internal unix sockets to a subdirectory of the Kong prefix.
+type: bugfix
+scope: Core

kong/clustering/utils.lua

@@ -24,8 +24,8 @@ local _log_prefix = "[clustering] "
 
 local KONG_VERSION = kong.version
 
-local prefix = kong.configuration.prefix or require("pl.path").abspath(ngx.config.prefix())
-local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/cluster_proxy_ssl_terminator.sock", prefix)
+local CLUSTER_PROXY_SSL_TERMINATOR_SOCK = fmt("unix:%s/cluster_proxy_ssl_terminator.sock",
+                                              kong.configuration.runtime_prefix)
 
 local _M = {}
 

kong/cmd/start.lua

@@ -13,11 +13,11 @@ local function is_socket(path)
   return lfs.attributes(path, "mode") == "socket"
 end
 
-local function cleanup_dangling_unix_sockets(prefix)
+local function cleanup_dangling_unix_sockets(runtime_prefix)
   local found = {}
 
-  for child in lfs.dir(prefix) do
-    local path = prefix .. "/" .. child
+  for child in lfs.dir(runtime_prefix) do
+    local path = runtime_prefix .. "/" .. child
     if is_socket(path) then
       table.insert(found, path)
     end
@@ -27,11 +27,11 @@ local function cleanup_dangling_unix_sockets(prefix)
     return
   end
 
-  log.warn("Found dangling unix sockets in the prefix directory (%q) while " ..
+  log.warn("Found dangling unix sockets in the runtime prefix (%q) while " ..
            "preparing to start Kong. This may be a sign that Kong was " ..
            "previously shut down uncleanly or is in an unknown state and " ..
            "could require further investigation.",
-           prefix)
+           runtime_prefix)
 
   log.warn("Attempting to remove dangling sockets before starting Kong...")
 
@@ -59,7 +59,7 @@ local function execute(args)
   assert(prefix_handler.prepare_prefix(conf, args.nginx_conf, nil, nil,
          args.nginx_conf_flags))
 
-  cleanup_dangling_unix_sockets(conf.prefix)
+  cleanup_dangling_unix_sockets(conf.runtime_prefix)
 
   _G.kong = kong_global.new()
   kong_global.init_pdk(_G.kong, conf)

kong/cmd/utils/prefix_handler.lua

@@ -481,6 +481,13 @@ local function prepare_prefix(kong_config, nginx_custom_template_path, skip_writ
     return nil, kong_config.prefix .. " is not a directory"
   end
 
+  if not exists(kong_config.runtime_prefix) then
+    local ok, err = makepath(kong_config.runtime_prefix)
+    if not ok then
+      return nil, err
+    end
+  end
+
   -- create directories in prefix
   for _, dir in ipairs {"logs", "pids"} do
     local ok, err = makepath(join(kong_config.prefix, dir))

kong/conf_loader/init.lua

@@ -482,6 +482,10 @@ local function load(path, custom_conf, opts)
   -- load absolute paths
   conf.prefix = abspath(conf.prefix)
 
+  -- the runtime prefix is where we keep listening unix sockets for IPC and
+  -- private APIs
+  conf.runtime_prefix = pl_path.join(conf.prefix, "runtime")
+
   if conf.lua_ssl_trusted_certificate
      and #conf.lua_ssl_trusted_certificate > 0 then
 

kong/global.lua

@@ -168,28 +168,20 @@ function _GLOBAL.init_pdk(self, kong_config)
 end
 
 
-function _GLOBAL.init_worker_events()
+function _GLOBAL.init_worker_events(kong_config)
   -- Note: worker_events will not work correctly if required at the top of the file.
   --       It must be required right here, inside the init function
   local worker_events
   local opts
 
-  local configuration = kong.configuration
-
-  -- `kong.configuration.prefix` is already normalized to an absolute path,
-  -- but `ngx.config.prefix()` is not
-  local prefix = configuration and
-                 configuration.prefix or
-                 require("pl.path").abspath(ngx.config.prefix())
-
+  local runtime_prefix = kong_config.runtime_prefix
   local sock = ngx.config.subsystem == "stream" and
                "stream_worker_events.sock" or
                "worker_events.sock"
 
-  local listening = "unix:" .. prefix .. "/" .. sock
+  local listening = "unix:" .. runtime_prefix .. "/" .. sock
 
-  local max_payload_len = configuration and
-                          configuration.worker_events_max_payload
+  local max_payload_len = kong_config.worker_events_max_payload
 
   if max_payload_len and max_payload_len > 65535 then   -- default is 64KB
     ngx.log(ngx.WARN,
@@ -203,9 +195,9 @@ function _GLOBAL.init_worker_events()
     listening = listening,  -- unix socket for broker listening
     max_queue_len = 1024 * 50,  -- max queue len for events buffering
     max_payload_len = max_payload_len,  -- max payload size in bytes
-    enable_privileged_agent = configuration and configuration.dedicated_config_processing
-                                            and configuration.role == "data_plane"
-                                             or false
+    enable_privileged_agent = kong_config.dedicated_config_processing
+                          and kong_config.role == "data_plane"
+                           or false,
   }
 
   worker_events = require "resty.events.compat"

kong/init.lua

@@ -837,7 +837,7 @@ function Kong.init_worker()
 
   schema_state = nil
 
-  local worker_events, err = kong_global.init_worker_events()
+  local worker_events, err = kong_global.init_worker_events(kong.configuration)
   if not worker_events then
     stash_init_worker_error("failed to instantiate 'kong.worker_events' " ..
                             "module: " .. err)

kong/runloop/events.lua

@@ -507,12 +507,17 @@ local stream_reconfigure_listener
 do
   local buffer = require "string.buffer"
 
-  -- `kong.configuration.prefix` is already normalized to an absolute path,
-  -- but `ngx.config.prefix()` is not
-  local PREFIX = kong and kong.configuration and
-                 kong.configuration.prefix or
-                 require("pl.path").abspath(ngx.config.prefix())
-  local STREAM_CONFIG_SOCK = "unix:" .. PREFIX .. "/stream_config.sock"
+  -- this module may be loaded before `kong.configuration` is initialized
+  local runtime_prefix = kong and kong.configuration
+                         and kong.configuration.runtime_prefix
+
+  if not runtime_prefix then
+    -- `kong.configuration.runtime_prefix` is already normalized to an absolute
+    -- path, but `ngx.config.prefix()` is not
+    runtime_prefix = require("pl.path").abspath(ngx.config.prefix() .. "/runtime")
+  end
+
+  local STREAM_CONFIG_SOCK = "unix:" .. runtime_prefix .. "/stream_config.sock"
   local IS_HTTP_SUBSYSTEM  = ngx.config.subsystem == "http"
 
   local function broadcast_reconfigure_event(data)

kong/runloop/handler.lua

@@ -896,11 +896,9 @@ return {
 
   init_worker = {
     before = function()
-      -- TODO: PR #9337 may affect the following line
-      local prefix = kong.configuration.prefix or ngx.config.prefix()
-
-      STREAM_TLS_TERMINATE_SOCK = fmt("unix:%s/stream_tls_terminate.sock", prefix)
-      STREAM_TLS_PASSTHROUGH_SOCK = fmt("unix:%s/stream_tls_passthrough.sock", prefix)
+      local runtime_prefix = kong.configuration.runtime_prefix
+      STREAM_TLS_TERMINATE_SOCK = fmt("unix:%s/stream_tls_terminate.sock", runtime_prefix)
+      STREAM_TLS_PASSTHROUGH_SOCK = fmt("unix:%s/stream_tls_passthrough.sock", runtime_prefix)
 
       log_level.init_worker()
 

kong/templates/nginx.lua

@@ -83,7 +83,7 @@ stream {
 
 > if cluster_ssl_tunnel then
     server {
-        listen unix:${{PREFIX}}/cluster_proxy_ssl_terminator.sock;
+        listen unix:${{RUNTIME_PREFIX}}/cluster_proxy_ssl_terminator.sock;
 
         proxy_pass ${{cluster_ssl_tunnel}};
         proxy_ssl on;

kong/templates/nginx_kong.lua

@@ -592,7 +592,7 @@ server {
 server {
     charset UTF-8;
     server_name kong_worker_events;
-    listen unix:${{PREFIX}}/worker_events.sock;
+    listen unix:${{RUNTIME_PREFIX}}/worker_events.sock;
     access_log off;
     location / {
         content_by_lua_block {

kong/templates/nginx_kong_stream.lua

@@ -94,7 +94,7 @@ server {
 > end
 
 > if stream_proxy_ssl_enabled then
-    listen unix:${{PREFIX}}/stream_tls_terminate.sock ssl proxy_protocol;
+    listen unix:${{RUNTIME_PREFIX}}/stream_tls_terminate.sock ssl proxy_protocol;
 > end
 
     access_log ${{PROXY_STREAM_ACCESS_LOG}};
@@ -175,7 +175,7 @@ server {
 }
 
 server {
-    listen unix:${{PREFIX}}/stream_tls_passthrough.sock proxy_protocol;
+    listen unix:${{RUNTIME_PREFIX}}/stream_tls_passthrough.sock proxy_protocol;
 
     access_log ${{PROXY_STREAM_ACCESS_LOG}};
     error_log ${{PROXY_STREAM_ERROR_LOG}} ${{LOG_LEVEL}};
@@ -205,7 +205,7 @@ server {
 
 > if database == "off" then
 server {
-    listen unix:${{PREFIX}}/stream_config.sock;
+    listen unix:${{RUNTIME_PREFIX}}/stream_config.sock;
 
     error_log  ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}};
 
@@ -216,7 +216,7 @@ server {
 > end -- database == "off"
 
 server {        # ignore (and close }, to ignore content)
-    listen unix:${{PREFIX}}/stream_rpc.sock;
+    listen unix:${{RUNTIME_PREFIX}}/stream_rpc.sock;
     error_log  ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}};
     content_by_lua_block {
         Kong.stream_api()
@@ -225,7 +225,7 @@ server {        # ignore (and close }, to ignore content)
 > end -- #stream_listeners > 0
 
 server {
-    listen unix:${{PREFIX}}/stream_worker_events.sock;
+    listen unix:${{RUNTIME_PREFIX}}/stream_worker_events.sock;
     error_log  ${{ADMIN_ERROR_LOG}} ${{LOG_LEVEL}};
     access_log off;
     content_by_lua_block {

kong/tools/stream_api.lua

@@ -37,7 +37,8 @@ local MAX_DATA_LEN = 2^22 - 1
 
 local HEADER_LEN = #st_pack(PACK_F, MAX_KEY_LEN, MAX_DATA_LEN)
 
-local SOCKET_PATH = "unix:" .. ngx.config.prefix() .. "/stream_rpc.sock"
+-- this module may be loaded before `kong.configuration` is initialized
+local SOCKET_PATH = "unix:" .. ngx.config.prefix() .. "/runtime/stream_rpc.sock"
 
 local stream_api = {}
 

spec/01-unit/01-db/11-declarative_lmdb_spec.lua

@@ -187,7 +187,7 @@ describe("#off preserve nulls", function()
     kong.configuration = kong_config
     kong.worker_events = kong.worker_events or
                          kong.cache and kong.cache.worker_events or
-                         assert(kong_global.init_worker_events())
+                         assert(kong_global.init_worker_events(kong.configuration))
     kong.cluster_events = kong.cluster_events or
                           kong.cache and kong.cache.cluster_events or
                           assert(kong_global.init_cluster_events(kong.configuration, kong.db))

spec/01-unit/03-conf_loader_spec.lua

@@ -2395,6 +2395,7 @@ describe("Configuration loader", function()
       local FIELDS = {
         -- CONF_BASIC
         prefix = true,
+        runtime_prefix = true,
         vaults = true,
         database = true,
         lmdb_environment_path = true,

spec/02-integration/02-cmd/02-start_stop_spec.lua

@@ -10,12 +10,14 @@ local read_file = helpers.file.read
 
 
 local PREFIX = helpers.test_conf.prefix
+local RUNTIME_PREFIX = helpers.test_conf.runtime_prefix
 local TEST_CONF = helpers.test_conf
 local TEST_CONF_PATH = helpers.test_conf_path
 
 
 local function wait_until_healthy(prefix)
   prefix = prefix or PREFIX
+  local runtime_prefix = prefix .. "/runtime"
 
   local cmd
 
@@ -41,11 +43,11 @@ local function wait_until_healthy(prefix)
   local conf = assert(helpers.get_running_conf(prefix))
 
   if conf.proxy_listen and conf.proxy_listen ~= "off" then
-    helpers.wait_for_file("socket", prefix .. "/worker_events.sock")
+    helpers.wait_for_file("socket", runtime_prefix .. "/worker_events.sock")
   end
 
   if conf.stream_listen and conf.stream_listen ~= "off" then
-    helpers.wait_for_file("socket", prefix .. "/stream_worker_events.sock")
+    helpers.wait_for_file("socket", runtime_prefix .. "/stream_worker_events.sock")
   end
 
   if conf.admin_listen and conf.admin_listen ~= "off" then
@@ -1034,11 +1036,51 @@ describe("kong start/stop #" .. strategy, function()
     end)
   end)
 
+  describe("runtime_prefix", function()
+    it("is created on demand by `kong prepare`", function()
+      local dir, cleanup = helpers.make_temp_dir()
+      finally(cleanup)
+
+      local cmd = fmt("prepare -p %q", dir)
+      assert.truthy(kong_exec(cmd), "expected '" .. cmd .. "' to succeed")
+      assert.truthy(helpers.path.isdir(dir .. "/runtime"),
+                    "expected '" .. dir .. "/runtime' directory to be created")
+    end)
+
+    it("can be a user-created symlink", function()
+      local prefix, cleanup = helpers.make_temp_dir()
+      finally(cleanup)
+
+      local runtime_prefix
+      runtime_prefix, cleanup = helpers.make_temp_dir()
+      finally(cleanup)
+
+      assert.truthy(helpers.execute(fmt("ln -sf %q %q/runtime", runtime_prefix, prefix)),
+                    "failed to symlink runtime prefix")
+
+      local preserve_prefix = true
+      assert(helpers.start_kong({
+        prefix = prefix,
+        database = "off",
+        nginx_conf = "spec/fixtures/custom_nginx.template",
+      }, nil, preserve_prefix))
+
+      finally(function()
+        helpers.stop_kong(prefix)
+      end)
+
+      wait_until_healthy(prefix)
+
+      assert.truthy(helpers.path.exists(runtime_prefix .. "/worker_events.sock"),
+                    "worker events socket was not created in the runtime_prefix dir")
+    end)
+  end)
+
   describe("dangling socket cleanup", function()
     local pidfile = TEST_CONF.nginx_pid
 
     -- the worker events socket is just one of many unix sockets we use
-    local event_sock = PREFIX .. "/worker_events.sock"
+    local event_sock = RUNTIME_PREFIX .. "/worker_events.sock"
 
     local env = {
       prefix                      = PREFIX,
@@ -1133,8 +1175,8 @@ describe("kong start/stop #" .. strategy, function()
     it("removes unix socket files in the prefix directory", function()
       local _, stderr = assert_start()
 
-      assert.matches("[warn] Found dangling unix sockets in the prefix directory", stderr, nil, true)
-      assert.matches(PREFIX, stderr, nil, true)
+      assert.matches("[warn] Found dangling unix sockets in the runtime prefix", stderr, nil, true)
+      assert.matches(RUNTIME_PREFIX, stderr, nil, true)
 
       assert.matches("removing unix socket", stderr)
       assert.matches(event_sock, stderr, nil, true)
@@ -1175,6 +1217,7 @@ describe("kong start/stop #" .. strategy, function()
 
     it("works with resty.events when KONG_PREFIX is a relative path", function()
       local prefix = "relpath"
+      local runtime_prefix = "relpath/runtime"
 
       finally(function()
         -- this test uses a non-default prefix, so it must manage
@@ -1201,8 +1244,8 @@ describe("kong start/stop #" .. strategy, function()
       -- wait until everything is running
       wait_until_healthy(prefix)
 
-      assert.truthy(helpers.path.exists(prefix .. "/worker_events.sock"))
-      assert.truthy(helpers.path.exists(prefix .. "/stream_worker_events.sock"))
+      assert.truthy(helpers.path.exists(runtime_prefix .. "/worker_events.sock"))
+      assert.truthy(helpers.path.exists(runtime_prefix .. "/stream_worker_events.sock"))
 
       local log = prefix .. "/logs/error.log"
       assert.logfile(log).has.no.line("[error]", true, 0)

spec/02-integration/03-db/14-dao_spec.lua

@@ -85,7 +85,7 @@ for _, strategy in helpers.all_strategies() do
         local kong_global = require("kong.global")
         local kong = _G.kong
 
-        kong.worker_events = assert(kong_global.init_worker_events())
+        kong.worker_events = assert(kong_global.init_worker_events(kong.configuration))
         kong.cluster_events = assert(kong_global.init_cluster_events(kong.configuration, kong.db))
         kong.cache = assert(kong_global.init_cache(kong.configuration, kong.cluster_events, kong.worker_events))
         kong.core_cache = assert(kong_global.init_core_cache(kong.configuration, kong.cluster_events, kong.worker_events))