Extract the main APIcast nginx configuration into own file

.luacheckrc

@@ -1,6 +1,18 @@
-std = 'ngx_lua+lua52' -- lua52 has table.pack
+local standards = require "luacheck.standards"
+
+-- stds global is provided by luacheck
+-- https://luacheck.readthedocs.io/en/stable/config.html#custom-sets-of-globals
+stds.ngx = {
+    read_globals = {
+        coroutine = standards.def_fields('_yield', '_create', '_resume')
+    }
+}
+
+std = 'ngx_lua+lua52+ngx' -- lua52 has table.pack
 
 busted = {std = "+busted", globals = { 'fixture' } }
 files["**/spec/**/*_spec.lua"] = busted
 
 globals = { 'rawlen' }
+
+files['gateway/config/*.lua'] = { globals = { 'context' } }

CHANGELOG.md

@@ -6,6 +6,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [Unreleased]
 
+### Added
+
+- Environment files now can use global `context` variable to share data [PR #964](https://github.com/3scale/apicast/pull/964)
+
+### Changed
+
+- Improve startup time by improving templating performance and caching filesystem access [PR #964](https://github.com/3scale/apicast/pull/964)
+- Liquid `default` filter now does not override `false` values [PR #964](https://github.com/3scale/apicast/pull/964)
+
 ### Fixed
 
 - Fix 3scale Batcher policy failing to cache and report requests containing app ID only [PR #956](https://github.com/3scale/apicast/pull/956), [THREESCALE-1515](https://issues.jboss.org/browse/THREESCALE-1515)

gateway/Roverfile.lock

@@ -3,7 +3,7 @@ busted 2.0.rc12-1||testing
 dkjson 2.5-2||testing
 inspect 3.1.1-0||production
 ldoc 1.4.6-2||development
-liquid 0.1.0-1||production
+liquid 0.1.3-1||production
 ljsonschema 0.1.0-1||testing
 lua-resty-env 0.4.0-1||production
 lua-resty-execvp 0.1.1-1||production

gateway/conf/nginx.conf.liquid

@@ -46,7 +46,6 @@ http {
   lua_socket_pool_size {{ lua_socket_pool_size | default: 512 }};
   server_names_hash_bucket_size 128;
 
-  log_format time '[$time_local] $host:$server_port $remote_addr:$remote_port "$request" $status $body_bytes_sent ($request_time) $post_action_impact';
   access_log off;
 
   lua_package_path "{{ lua_path | default: package.path }}";
@@ -69,120 +68,19 @@ http {
   {%- endcapture -%}
 
   {% for file in "http.d/*.conf" | filesystem %}
+    ## include {{ file }}
     {% include file %}
+    ## end {{ file }}
   {% endfor %}
 
   {% if opentracing_tracer != empty %}
     {%- capture tracer_conf %}conf.d/opentracing/{{ opentracing_tracer }}.conf.liquid{%- endcapture -%}
     {% include tracer_conf %}
   {% endif %}
 
-  server {
-    listen {{ port.management | default: 8090 }};
-    server_name {{ server_name.management | default: 'management _' }};
-
-    {% if opentracing_tracer != empty %}
-    opentracing_operation_name "apicast_management";
-    opentracing_trace_locations off;
-    {% endif %}
-
-    {% include "conf.d/management.conf" %}
-  }
-
-  server {
-    listen {{ port.backend | default: 8081 }};
-    server_name backend;
-
-    {% if opentracing_tracer != empty %}
-    opentracing_operation_name "apicast_mockbackend";
-    opentracing_trace_locations off;
-    {% endif %}
-
-    {% include "conf.d/backend.conf" %}
-  }
-
-  upstream echo {
-    server 127.0.0.1:{{ port.echo | default: 8081 }};
-    keepalive 1024;
-  }
-
-  server {
-    listen {{ port.echo | default: 8081 }} default_server;
-    server_name echo _;
-
-    {% if opentracing_tracer != empty %}
-    opentracing_operation_name "apicast_echo";
-    opentracing_trace_locations off;
-    {% endif %}
-
-    {% include "conf.d/echo.conf" %}
-  }
-
-  server {
-
-    set $access_logs_enabled '1';
-    access_log {{ access_log_file | default: "/dev/stdout" }} time if=$access_logs_enabled;
-
-    {%- assign http_port = port.apicast | default: 8080 %}
-    {%- assign https_port = env.APICAST_HTTPS_PORT %}
-
-    {% if http_port != https_port -%}
-      listen {{ http_port }};
-    {% endif %}
-
-    {% if https_port -%}
-    listen {{ https_port }} ssl;
-
-    {%- assign https_certificate = env.APICAST_HTTPS_CERTIFICATE -%}
-    ssl_certificate {% if https_certificate -%}
-      {{  https_certificate }}
-    {%- else -%}
-      {{ "conf/server.crt" | filesystem | first }}
-    {%- endif %};
-
-    {%- assign https_certificate_key = env.APICAST_HTTPS_CERTIFICATE_KEY -%}
-    ssl_certificate_key {% if https_certificate_key -%}
-      {{  https_certificate_key }}
-    {%- else -%}
-      {{ "conf/server.key" | filesystem | first }}
-    {%- endif %};
-
-    ssl_verify_client optional_no_ca;
-    ssl_certificate_by_lua_block { require('apicast.executor'):ssl_certificate() }
-    {%- endif %}
-
-    server_name _;
-
-    {% if opentracing_tracer != empty %}
-    opentracing_operation_name "apicast";
-    opentracing_trace_locations on;
-    {% endif %}
-
-    {% include "http.d/ssl.conf" %}
-
-    {% for file in "apicast.d/*.conf" | filesystem %}
-      {% include file %}
-    {% endfor %}
-    {% include "conf.d/apicast.conf" %}
-  }
-
-  {% if port.metrics %}
-    lua_shared_dict prometheus_metrics 16M;
-    server {
-      access_log off;
-      listen {{ port.metrics }};
-      server_name metrics prometheus _;
-
-      location /metrics {
-        content_by_lua_block { require('apicast.executor'):metrics() }
-      }
-
-      location /nginx_status {
-        internal;
-        stub_status;
-      }
-    }
-  {% endif %}
+  {% for file in template | default: 'http.d/apicast.conf.liquid' | filesystem %}
+    {% include file %}
+  {% endfor %}
 
   lua_shared_dict limiter 1m;
 

gateway/http.d/apicast.conf.liquid

@@ -0,0 +1,108 @@
+log_format time '[$time_local] $host:$server_port $remote_addr:$remote_port "$request" $status $body_bytes_sent ($request_time) $post_action_impact';
+
+server {
+  listen {{ port.management | default: 8090 }};
+  server_name {{ server_name.management | default: 'management _' }};
+
+  {% if opentracing_tracer != empty %}
+  opentracing_operation_name "apicast_management";
+  opentracing_trace_locations off;
+  {% endif %}
+
+  {% include "conf.d/management.conf" %}
+}
+
+server {
+  listen {{ port.backend | default: 8081 }};
+  server_name backend;
+
+  {% if opentracing_tracer != empty %}
+  opentracing_operation_name "apicast_mockbackend";
+  opentracing_trace_locations off;
+  {% endif %}
+
+  {% include "conf.d/backend.conf" %}
+}
+
+upstream echo {
+  server 127.0.0.1:{{ port.echo | default: 8081 }};
+  keepalive 1024;
+}
+
+server {
+  listen {{ port.echo | default: 8081 }} default_server;
+  server_name echo _;
+
+  {% if opentracing_tracer != empty %}
+  opentracing_operation_name "apicast_echo";
+  opentracing_trace_locations off;
+  {% endif %}
+
+  {% include "conf.d/echo.conf" %}
+}
+
+server {
+
+  set $access_logs_enabled '1';
+  access_log {{ access_log_file | default: "/dev/stdout" }} time if=$access_logs_enabled;
+
+  {%- assign http_port = port.apicast | default: 8080 %}
+  {%- assign https_port = env.APICAST_HTTPS_PORT %}
+
+  {% if http_port != https_port -%}
+    listen {{ http_port }};
+  {% endif %}
+
+  {% if https_port -%}
+  listen {{ https_port }} ssl;
+
+  {%- assign https_certificate = env.APICAST_HTTPS_CERTIFICATE -%}
+  ssl_certificate {% if https_certificate -%}
+    {{  https_certificate }}
+  {%- else -%}
+    {{ "conf/server.crt" | filesystem | first }}
+  {%- endif %};
+
+  {%- assign https_certificate_key = env.APICAST_HTTPS_CERTIFICATE_KEY -%}
+  ssl_certificate_key {% if https_certificate_key -%}
+    {{  https_certificate_key }}
+  {%- else -%}
+    {{ "conf/server.key" | filesystem | first }}
+  {%- endif %};
+
+  ssl_verify_client optional_no_ca;
+  ssl_certificate_by_lua_block { require('apicast.executor'):ssl_certificate() }
+  {%- endif %}
+
+  server_name _;
+
+  {% if opentracing_tracer != empty %}
+  opentracing_operation_name "apicast";
+  opentracing_trace_locations on;
+  {% endif %}
+
+  {% include "http.d/ssl.conf" %}
+
+  {% for file in "apicast.d/*.conf" | filesystem %}
+    {% include file %}
+  {% endfor %}
+  {% include "conf.d/apicast.conf" %}
+}
+
+{% if port.metrics %}
+  lua_shared_dict prometheus_metrics 16M;
+  server {
+    access_log off;
+    listen {{ port.metrics }};
+    server_name metrics prometheus _;
+
+    location /metrics {
+      content_by_lua_block { require('apicast.executor'):metrics() }
+    }
+
+    location /nginx_status {
+      internal;
+      stub_status;
+    }
+  }
+{% endif %}

gateway/src/apicast/cli/command/start.lua

@@ -81,7 +81,7 @@ end
 
 
 local function build_environment_config(options)
-    local config = Environment.new()
+    local config = Environment.new(options)
 
     resty_env.set('APICAST_POLICY_LOAD_PATH', concat(options.policy_load_path,':'))
 
@@ -139,6 +139,9 @@ local function build_context(options, config)
 
     context.access_log_file = options.access_log_file
 
+    -- expose parsed CLI options
+    context.options = options
+
     return context
 end
 

gateway/src/apicast/cli/environment.lua

@@ -117,6 +117,7 @@ _M.default_config = {
     policy_chain = require('apicast.policy_chain').default(),
     nameservers = parse_nameservers(),
     worker_processes = cpus() or 'auto',
+    template = 'http.d/apicast.conf.liquid',
     package = {
         path = package.path,
         cpath = package.cpath,
@@ -151,8 +152,8 @@ end
 
 --- Initialize new environment.
 -- @treturn Environment
-function _M.new()
-    return setmetatable({ _context = linked_list.readonly(_M.default_config), loaded = {} }, mt)
+function _M.new(context)
+    return setmetatable({ _context = linked_list.readonly(_M.default_config, context), loaded = {} }, mt)
 end
 
 local function expand_environment_name(name)

gateway/src/apicast/cli/template.lua

@@ -77,6 +77,17 @@ local function nginx_prefix()
     if match then return match[1] end
 end
 
+local function dirtree(dir, cache)
+  local cached = cache[dir]
+
+  if cached then
+    return pairs(cached)
+  else
+    cache[dir] = {}
+    return fs(dir)
+  end
+end
+
 function _M:interpret(str)
     local interpreter = build_interpreter(str)
 
@@ -85,27 +96,31 @@ function _M:interpret(str)
     local filter_set = FilterSet:new()
     local resource_limit = ResourceLimit:new(nil, 1000, nil)
 
+    local filesystem_cache = {}
+
     filter_set:add_filter('filesystem', function(pattern)
         local files = {}
         local included = {}
 
         for _, root in ipairs({ self.root, pl.path.currentdir(), ngx.config.prefix(), nginx_prefix() }) do
-            for filename in fs(root) do
+            for filename in dirtree(root, filesystem_cache) do
                 local file = pl.path.relpath(filename, root)
 
                 if pl.dir.fnmatch(file, pattern) and not included[filename] and not included[file] then
                     insert(files, filename)
                     included[filename] = true
                     included[file] = true
                 end
+
+                filesystem_cache[root][filename] = true
             end
         end
 
         return files
     end)
 
     filter_set:add_filter('default', function(value, default)
-        return value or default
+        if value == nil then return default else return value end
     end)
 
     filter_set:add_filter('starts_with', function(string, ...)

spec/cli/environment_spec.lua

@@ -0,0 +1,14 @@
+local _M = require('apicast.cli.environment')
+
+describe('Environment Configuration', function ()
+
+  describe('.new', function()
+    it('accepts default', function()
+      local default = { foo = 'bar' }
+      local env = _M.new(default)
+
+      assert.contains(default, env:context())
+    end)
+  end)
+
+end)