Merge pull request #590 from 3scale/env-configuration-ineritance

[cli] decouple apicast environment and 3scale configuration channel

CHANGELOG.md

@@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - SOAP policy [PR #567](https://github.com/3scale/apicast/pull/567)
 - Ability to set custom directories to load policies from [PR #581](https://github.com/3scale/apicast/pull/581)
 - CLI is running with proper log level set by `APICAST_LOG_LEVEL` [PR #585](https://github.com/3scale/apicast/pull/585)
+- 3scale configuration (staging/production) can be passed as `-3` or `--channel` on the CLI [PR #590](https://github.com/3scale/apicast/pull/590)
+- APIcast CLI loads environments defined by `APICAST_ENVIRONMENT` variable [PR #590](https://github.com/3scale/apicast/pull/590)
 
 ## Fixed
 
@@ -53,6 +55,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Sandbox loading policies [PR #566](https://github.com/3scale/apicast/pull/566)
 - Extracted `usage` and `mapping_rules_matcher` modules so they can be used from policies [PR #580](https://github.com/3scale/apicast/pull/580)
 - Renamed all `apicast/policy/*/policy.lua` to `apicast/policy/*/init.lua` to match Lua naming [PR #579](https://github.com/3scale/apicast/pull/579)
+- Environment configuration can now define the configuration loader or cache [PR #590](https://github.com/3scale/apicast/pull/590).
+- APIcast starts with "boot" configuration loader by default (because production is the default environment) [PR #590](https://github.com/3scale/apicast/pull/590).
 
 ## [3.2.0-alpha2] - 2017-11-30
 

Makefile

@@ -121,7 +121,7 @@ gateway-logs:
 
 test-runtime-image: export IMAGE_NAME = apicast-runtime-test
 test-runtime-image: runtime-image clean-containers ## Smoke test the runtime image. Pass any docker image in IMAGE_NAME parameter.
-	$(DOCKER_COMPOSE) run --rm --user 100001 gateway apicast -d
+	$(DOCKER_COMPOSE) run --rm --user 100001 gateway apicast -l -d
 	@echo -e $(SEPARATOR)
 	$(DOCKER_COMPOSE) run --rm --user 100002 -e APICAST_CONFIGURATION_LOADER=boot -e THREESCALE_PORTAL_ENDPOINT=https://echo-api.3scale.net gateway bin/apicast -d
 	@echo -e $(SEPARATOR)

doc/parameters.md

@@ -191,3 +191,13 @@ before the client is throttled by adding latency.
 
 Double colon (`:`) separated list of paths where APIcast should look for policies.
 It can be used to first load policies from a development directory or to load examples.
+
+### `APICAST_ENVIRONMENT`
+
+**Default**:
+**Value:**: string\[:<string>\]
+**Example**: production:cloud-hosted
+
+Double colon (`:`) separated list of environments (or paths) APIcast should load.
+It can be used instead of `-e` or `---environment` parameter on the CLI and for example
+stored in the container image as default environment. Any value passed on the CLI overrides this variable.

docker-compose.yml

@@ -4,9 +4,20 @@ services:
     image: ${IMAGE_NAME:-apicast-test}
     depends_on:
     - redis
+    - echo
     env_file: .env
+    environment:
+      THREESCALE_PORTAL_ENDPOINT: http://echo:8081/config/
+  echo:
+    image: ${IMAGE_NAME:-apicast-test}
+    environment:
+      APICAST_CONFIGURATION_LOADER: test
+      APICAST_MANAGEMENT_API: debug
+    command: bin/apicast
+    ports:
+      - '8081'
   dev:
-    image: ${IMAGE_NAME}
+    image: ${IMAGE_NAME:-apicast-test}
     depends_on:
     - redis
     ports:
@@ -28,6 +39,8 @@ services:
     dns: 127.0.0.1
     environment:
       APICAST_MANAGEMENT_API: debug
+      APICAST_LOG_LEVEL: debug
+      APICAST_CONFIGURATION_LOADER: test
     dns_search:
       - example.com
   prove:
@@ -42,7 +55,7 @@ services:
     depends_on:
       - redis
     volumes_from:
-    - container:${COMPOSE_PROJECT_NAME}-source
+    - container:${COMPOSE_PROJECT_NAME:-apicast_build_0}-source
   redis:
     image: redis
   keycloak:

gateway/conf.d/echo.conf

@@ -19,3 +19,7 @@ location / {
       end
     }
 }
+
+location /config/ {
+  echo "{}";
+}

gateway/config/development.lua

@@ -2,4 +2,6 @@ return {
     worker_processes = '1',
     master_process = 'off',
     lua_code_cache = 'off',
+    configuration_loader = 'lazy',
+    configuration_cache = 0,
 }

gateway/config/production.lua

@@ -1,4 +1,6 @@
 return {
     master_process = 'on',
     lua_code_cache = 'on',
+    configuration_loader = 'boot',
+    configuration_cache = os.getenv('APICAST_CONFIGURATION_CACHE') or 5*60,
 }

gateway/config/sandbox.lua

@@ -0,0 +1 @@
+staging.lua

gateway/config/staging.lua

@@ -0,0 +1,6 @@
+return {
+    master_process = 'on',
+    lua_code_cache = 'on',
+    configuration_loader = 'lazy',
+    configuration_cache = 0,
+}

gateway/cpanfile

@@ -1,2 +1,2 @@
-requires 'Test::APIcast', '0.07';
+requires 'Test::APIcast', '0.08';
 requires 'Crypt::JWT';

gateway/http.d/init.conf

@@ -15,7 +15,7 @@ init_by_lua_block {
     -- This ENV variable is defined in the main nginx.conf.liquid and injected when including this partial.
     -- The content of the ENV variable is a Lua table, so when rendered it actually can run ipairs on it.
     for k,v in pairs({{ ENV }}) do
-      if type(k) == 'string' and k ~= 'APICAST_CONFIGURATION_LOADER' then
+      if type(k) == 'string' and not resty_env.value(k) then
         resty_env.set(k,v)
       end
     end

gateway/libexec/run

@@ -18,7 +18,16 @@ lua_ssl_verify_depth 5;
 lua_ssl_trusted_certificate "${ssl_cert_file}";
 _NGINX_
 
-exec 'resty',
+my @resty_args = (
     '--errlog-level', $errlog_level,
     '--http-include', $ssl_conf_file,
+);
+
+my $nginx = $ENV{APICAST_OPENRESTY_BINARY} || $ENV{TEST_NGINX_BINARY};
+if (defined $nginx) {
+    push @resty_args, '--nginx', $nginx;
+}
+
+exec 'resty',
+    @resty_args,
     $lua_file, @ARGV;

gateway/src/apicast/cli/command/start.lua

@@ -5,9 +5,12 @@ local max = math.max
 local insert = table.insert
 local concat = table.concat
 local format = string.format
+local tostring = tostring
+local tonumber = tonumber
 
 local exec = require('resty.execvp')
 local resty_env = require('resty.env')
+local re = require('ngx.re')
 
 local Template = require('apicast.cli.template')
 local Environment = require('apicast.cli.environment')
@@ -41,7 +44,7 @@ end
 
 local function update_env(env)
     for name, value in pairs(env) do
-        resty_env.set(name, value)
+        resty_env.set(name, tostring(value))
     end
 end
 
@@ -104,13 +107,13 @@ local function openresty_binary(candidates)
         find_openresty_command(candidates)
 end
 
-local function build_env(options, config)
+local function build_env(options, config, context)
     return {
         APICAST_CONFIGURATION = options.configuration,
-        APICAST_CONFIGURATION_LOADER = options.boot and 'boot' or 'lazy',
-        APICAST_CONFIGURATION_CACHE = options.cache,
-        THREESCALE_DEPLOYMENT_ENV = config.name,
-        APICAST_POLICY_LOAD_PATH = options.policy_load_path,
+        APICAST_CONFIGURATION_LOADER = tostring(options.configuration_loader or context.configuration_loader or 'lazy'),
+        APICAST_CONFIGURATION_CACHE = tostring(options.cache or context.configuration_cache or 0),
+        THREESCALE_DEPLOYMENT_ENV = context.configuration_channel or options.channel or config.name,
+        APICAST_POLICY_LOAD_PATH = options.policy_load_path or context.policy_load_path,
     }
 end
 
@@ -138,7 +141,7 @@ function mt:__call(options)
     local openresty = openresty_binary(self.openresty)
     local config = build_environment_config(options)
     local context = build_context(options, config)
-    local env = build_env(options, config)
+    local env = build_env(options, config, context)
 
     local template_path = options.template
 
@@ -164,15 +167,34 @@ function mt:__call(options)
     return exec(openresty, cmd, env)
 end
 
+local function split_by(pattern)
+  return function(str)
+    return re.split(str or '', pattern, 'oj')
+  end
+end
+
+local load_env = split_by(':')
+
 local function configure(cmd)
     cmd:usage("Usage: apicast-cli start [OPTIONS]")
     cmd:option("--template", "Nginx config template.", 'conf/nginx.conf.liquid')
 
+    local channel = resty_env.value('THREESCALE_DEPLOYMENT_ENV') or 'production'
+    local loaded_env = Environment.loaded()
+
+    insert(loaded_env, 1, channel)
 
-    cmd:option('-e --environment', "Deployment to start. Can also be a path to a Lua file.", resty_env.value('THREESCALE_DEPLOYMENT_ENV') or 'production'):count('*')
-    cmd:flag('--dev', 'Start in development environment')
+    cmd:option('-3 --channel', "3scale configuration channel to use.", channel):action(function(args, name, chan)
+      args.environment[1] = chan
+      args[name] = chan
+    end):count('0-1')
+    cmd:option('-e --environment', "Deployment to start. Can also be a path to a Lua file.", resty_env.value('APICAST_ENVIRONMENT'))
+      :count('*'):init(loaded_env):action('concat'):convert(load_env)
+    cmd:flag('--development --dev', 'Start in development environment'):action(function(arg, name)
+      insert(arg.environment, name)
+    end)
 
-    cmd:flag("-m --master", "Test the nginx config"):args('?')
+    cmd:flag("-m --master", "Control nginx master process.", 'on'):args('?')
     cmd:flag("-t --test", "Test the nginx config")
     cmd:flag("--debug", "Debug mode. Prints more information.")
     cmd:option("-c --configuration",
@@ -183,17 +205,26 @@ local function configure(cmd)
         "Number of worker processes to start.",
         resty_env.value('APICAST_WORKERS') or Environment.default_config.worker_processes)
     cmd:option("-p --pid", "Path to the PID file.")
-    cmd:mutex(
-        cmd:flag('-b --boot',
-            "Load configuration on boot.",
-            resty_env.value('APICAST_CONFIGURATION_LOADER') == 'boot'),
-        cmd:flag('-l --lazy',
-            "Load configuration on demand.",
-            resty_env.value('APICAST_CONFIGURATION_LOADER') == 'lazy')
-    )
+
+    do
+      local target = 'configuration_loader'
+      local configuration_loader = resty_env.value('APICAST_CONFIGURATION_LOADER')
+      local function set_configuration_loader(value)
+        return function(args) args[target] = value end
+      end
+
+      cmd:mutex(
+          cmd:flag('-b --boot',
+              "Load configuration on boot.",
+              configuration_loader == 'boot'):action(set_configuration_loader('boot')):target('configuration_loader'):init(configuration_loader),
+          cmd:flag('-l --lazy',
+              "Load configuration on demand.",
+              configuration_loader == 'lazy'):action(set_configuration_loader('lazy')):target('configuration_loader'):init(configuration_loader)
+      )
+    end
     cmd:option("-i --refresh-interval",
         "Cache configuration for N seconds. Using 0 will reload on every request (not for production).",
-        resty_env.value('APICAST_CONFIGURATION_CACHE'))
+        resty_env.value('APICAST_CONFIGURATION_CACHE')):convert(tonumber)
 
     cmd:option("--policy-load-path",
         "Load path where to find policies. Entries separated by `:`.",

gateway/src/apicast/cli/environment.lua

@@ -75,18 +75,23 @@ _M.default_config = {
 
 local mt = { __index = _M }
 
+--- Return loaded environments defined as environment variable.
+-- @treturn {string,...}
+function _M.loaded()
+    local value = resty_env.value('APICAST_LOADED_ENVIRONMENTS')
+    return re.split(value or '', [[\|]], 'jo')
+end
+
 --- Load an environment from files in ENV.
 -- @treturn Environment
 function _M.load()
-    local value = resty_env.value('APICAST_LOADED_ENVIRONMENTS')
     local env = _M.new()
+    local environments = _M.loaded()
 
-    if not value then
+    if not environments then
         return env
     end
 
-    local environments = re.split(value, '\\|', 'jo')
-
     for i=1,#environments do
         assert(env:add(environments[i]))
     end
@@ -143,7 +148,7 @@ function _M:add(env)
 
     local config = loadfile(path, 't', {
         print = print, inspect = require('inspect'), context = self._context,
-        tonumber = tonumber, tostring = tostring,
+        tonumber = tonumber, tostring = tostring, os = { getenv = resty_env.value },
         pcall = pcall, require = require, assert = assert, error = error,
     })
 

gateway/src/apicast/configuration_loader.lua

@@ -122,7 +122,7 @@ function boot.init(configuration)
   if config and init then
     ngx.log(ngx.DEBUG, 'downloaded configuration: ', config)
   else
-    ngx.log(ngx.EMERG, 'failed to load configuration, exiting (code ', code, ')\n',  err)
+    ngx.log(ngx.EMERG, 'failed to load configuration, exiting (code ', code, ')\n',  err or ngx.config.debug and debug.traceback())
     os.exit(1)
   end
 
@@ -213,8 +213,10 @@ function lazy.rewrite(configuration, host)
   return configuration
 end
 
+local test = { init = noop, init_worker = noop, rewrite = noop }
+
 local modes = {
-  boot = boot, lazy = lazy, default = 'lazy'
+  boot = boot, lazy = lazy, default = 'lazy', test = test
 }
 
 function _M.new(mode)

t/configuration-loading-boot-with-config.t

@@ -1,10 +1,8 @@
 use lib 't';
 use Test::APIcast::Blackbox 'no_plan';
 
-$ENV{APICAST_CONFIGURATION_LOADER} = 'boot';
-
-env_to_nginx(
-    'APICAST_CONFIGURATION_LOADER',
+env_to_apicast(
+    'APICAST_CONFIGURATION_LOADER' => 'boot'
 );
 
 log_level('warn');