diff --git a/CHANGELOG.md b/CHANGELOG.md index afb518d6a..5ca03949c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - SOAP policy [PR #567](https://github.com/3scale/apicast/pull/567) - Ability to set custom directories to load policies from [PR #581](https://github.com/3scale/apicast/pull/581) - CLI is running with proper log level set by `APICAST_LOG_LEVEL` [PR #585](https://github.com/3scale/apicast/pull/585) +- 3scale configuration (staging/production) can be passed as `-3` or `--channel` on the CLI [PR #590](https://github.com/3scale/apicast/pull/590) +- APIcast CLI loads environments defined by `APICAST_ENVIRONMENT` variable [PR #590](https://github.com/3scale/apicast/pull/590) ## Fixed @@ -53,6 +55,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Sandbox loading policies [PR #566](https://github.com/3scale/apicast/pull/566) - Extracted `usage` and `mapping_rules_matcher` modules so they can be used from policies [PR #580](https://github.com/3scale/apicast/pull/580) - Renamed all `apicast/policy/*/policy.lua` to `apicast/policy/*/init.lua` to match Lua naming [PR #579](https://github.com/3scale/apicast/pull/579) +- Environment configuration can now define the configuration loader or cache [PR #590](https://github.com/3scale/apicast/pull/590). +- APIcast starts with "boot" configuration loader by default (because production is the default environment) [PR #590](https://github.com/3scale/apicast/pull/590). ## [3.2.0-alpha2] - 2017-11-30 diff --git a/Makefile b/Makefile index b5907577d..f951b460e 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,7 @@ gateway-logs: test-runtime-image: export IMAGE_NAME = apicast-runtime-test test-runtime-image: runtime-image clean-containers ## Smoke test the runtime image. Pass any docker image in IMAGE_NAME parameter. - $(DOCKER_COMPOSE) run --rm --user 100001 gateway apicast -d + $(DOCKER_COMPOSE) run --rm --user 100001 gateway apicast -l -d @echo -e $(SEPARATOR) $(DOCKER_COMPOSE) run --rm --user 100002 -e APICAST_CONFIGURATION_LOADER=boot -e THREESCALE_PORTAL_ENDPOINT=https://echo-api.3scale.net gateway bin/apicast -d @echo -e $(SEPARATOR) diff --git a/doc/parameters.md b/doc/parameters.md index ea4cf2dc3..5444ca4bf 100644 --- a/doc/parameters.md +++ b/doc/parameters.md @@ -191,3 +191,13 @@ before the client is throttled by adding latency. Double colon (`:`) separated list of paths where APIcast should look for policies. It can be used to first load policies from a development directory or to load examples. + +### `APICAST_ENVIRONMENT` + +**Default**: +**Value:**: string\[:\] +**Example**: production:cloud-hosted + +Double colon (`:`) separated list of environments (or paths) APIcast should load. +It can be used instead of `-e` or `---environment` parameter on the CLI and for example +stored in the container image as default environment. Any value passed on the CLI overrides this variable. diff --git a/docker-compose.yml b/docker-compose.yml index 08f8ecc6a..99033fae6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,9 +4,20 @@ services: image: ${IMAGE_NAME:-apicast-test} depends_on: - redis + - echo env_file: .env + environment: + THREESCALE_PORTAL_ENDPOINT: http://echo:8081/config/ + echo: + image: ${IMAGE_NAME:-apicast-test} + environment: + APICAST_CONFIGURATION_LOADER: test + APICAST_MANAGEMENT_API: debug + command: bin/apicast + ports: + - '8081' dev: - image: ${IMAGE_NAME} + image: ${IMAGE_NAME:-apicast-test} depends_on: - redis ports: @@ -28,6 +39,8 @@ services: dns: 127.0.0.1 environment: APICAST_MANAGEMENT_API: debug + APICAST_LOG_LEVEL: debug + APICAST_CONFIGURATION_LOADER: test dns_search: - example.com prove: @@ -42,7 +55,7 @@ services: depends_on: - redis volumes_from: - - container:${COMPOSE_PROJECT_NAME}-source + - container:${COMPOSE_PROJECT_NAME:-apicast_build_0}-source redis: image: redis keycloak: diff --git a/gateway/conf.d/echo.conf b/gateway/conf.d/echo.conf index 96a811e2d..f858ff752 100644 --- a/gateway/conf.d/echo.conf +++ b/gateway/conf.d/echo.conf @@ -19,3 +19,7 @@ location / { end } } + +location /config/ { + echo "{}"; +} diff --git a/gateway/config/development.lua b/gateway/config/development.lua index c08503031..7e7d4d546 100644 --- a/gateway/config/development.lua +++ b/gateway/config/development.lua @@ -2,4 +2,6 @@ return { worker_processes = '1', master_process = 'off', lua_code_cache = 'off', + configuration_loader = 'lazy', + configuration_cache = 0, } diff --git a/gateway/config/production.lua b/gateway/config/production.lua index 3c60ffb0f..1e02da15f 100644 --- a/gateway/config/production.lua +++ b/gateway/config/production.lua @@ -1,4 +1,6 @@ return { master_process = 'on', lua_code_cache = 'on', + configuration_loader = 'boot', + configuration_cache = os.getenv('APICAST_CONFIGURATION_CACHE') or 5*60, } diff --git a/gateway/config/sandbox.lua b/gateway/config/sandbox.lua new file mode 120000 index 000000000..8c4b5039a --- /dev/null +++ b/gateway/config/sandbox.lua @@ -0,0 +1 @@ +staging.lua \ No newline at end of file diff --git a/gateway/config/staging.lua b/gateway/config/staging.lua new file mode 100644 index 000000000..ac99c7130 --- /dev/null +++ b/gateway/config/staging.lua @@ -0,0 +1,6 @@ +return { + master_process = 'on', + lua_code_cache = 'on', + configuration_loader = 'lazy', + configuration_cache = 0, +} diff --git a/gateway/cpanfile b/gateway/cpanfile index 8f37251af..1e8fa24a2 100644 --- a/gateway/cpanfile +++ b/gateway/cpanfile @@ -1,2 +1,2 @@ -requires 'Test::APIcast', '0.07'; +requires 'Test::APIcast', '0.08'; requires 'Crypt::JWT'; diff --git a/gateway/http.d/init.conf b/gateway/http.d/init.conf index 1676d7552..faab4b951 100644 --- a/gateway/http.d/init.conf +++ b/gateway/http.d/init.conf @@ -15,7 +15,7 @@ init_by_lua_block { -- This ENV variable is defined in the main nginx.conf.liquid and injected when including this partial. -- The content of the ENV variable is a Lua table, so when rendered it actually can run ipairs on it. for k,v in pairs({{ ENV }}) do - if type(k) == 'string' and k ~= 'APICAST_CONFIGURATION_LOADER' then + if type(k) == 'string' and not resty_env.value(k) then resty_env.set(k,v) end end diff --git a/gateway/libexec/run b/gateway/libexec/run index ee06f4674..074f51d09 100755 --- a/gateway/libexec/run +++ b/gateway/libexec/run @@ -18,7 +18,16 @@ lua_ssl_verify_depth 5; lua_ssl_trusted_certificate "${ssl_cert_file}"; _NGINX_ -exec 'resty', +my @resty_args = ( '--errlog-level', $errlog_level, '--http-include', $ssl_conf_file, +); + +my $nginx = $ENV{APICAST_OPENRESTY_BINARY} || $ENV{TEST_NGINX_BINARY}; +if (defined $nginx) { + push @resty_args, '--nginx', $nginx; +} + +exec 'resty', + @resty_args, $lua_file, @ARGV; diff --git a/gateway/src/apicast/cli/command/start.lua b/gateway/src/apicast/cli/command/start.lua index e98f4f393..e6842c037 100644 --- a/gateway/src/apicast/cli/command/start.lua +++ b/gateway/src/apicast/cli/command/start.lua @@ -5,9 +5,12 @@ local max = math.max local insert = table.insert local concat = table.concat local format = string.format +local tostring = tostring +local tonumber = tonumber local exec = require('resty.execvp') local resty_env = require('resty.env') +local re = require('ngx.re') local Template = require('apicast.cli.template') local Environment = require('apicast.cli.environment') @@ -41,7 +44,7 @@ end local function update_env(env) for name, value in pairs(env) do - resty_env.set(name, value) + resty_env.set(name, tostring(value)) end end @@ -104,13 +107,13 @@ local function openresty_binary(candidates) find_openresty_command(candidates) end -local function build_env(options, config) +local function build_env(options, config, context) return { APICAST_CONFIGURATION = options.configuration, - APICAST_CONFIGURATION_LOADER = options.boot and 'boot' or 'lazy', - APICAST_CONFIGURATION_CACHE = options.cache, - THREESCALE_DEPLOYMENT_ENV = config.name, - APICAST_POLICY_LOAD_PATH = options.policy_load_path, + APICAST_CONFIGURATION_LOADER = tostring(options.configuration_loader or context.configuration_loader or 'lazy'), + APICAST_CONFIGURATION_CACHE = tostring(options.cache or context.configuration_cache or 0), + THREESCALE_DEPLOYMENT_ENV = context.configuration_channel or options.channel or config.name, + APICAST_POLICY_LOAD_PATH = options.policy_load_path or context.policy_load_path, } end @@ -138,7 +141,7 @@ function mt:__call(options) local openresty = openresty_binary(self.openresty) local config = build_environment_config(options) local context = build_context(options, config) - local env = build_env(options, config) + local env = build_env(options, config, context) local template_path = options.template @@ -164,15 +167,34 @@ function mt:__call(options) return exec(openresty, cmd, env) end +local function split_by(pattern) + return function(str) + return re.split(str or '', pattern, 'oj') + end +end + +local load_env = split_by(':') + local function configure(cmd) cmd:usage("Usage: apicast-cli start [OPTIONS]") cmd:option("--template", "Nginx config template.", 'conf/nginx.conf.liquid') + local channel = resty_env.value('THREESCALE_DEPLOYMENT_ENV') or 'production' + local loaded_env = Environment.loaded() + + insert(loaded_env, 1, channel) - cmd:option('-e --environment', "Deployment to start. Can also be a path to a Lua file.", resty_env.value('THREESCALE_DEPLOYMENT_ENV') or 'production'):count('*') - cmd:flag('--dev', 'Start in development environment') + cmd:option('-3 --channel', "3scale configuration channel to use.", channel):action(function(args, name, chan) + args.environment[1] = chan + args[name] = chan + end):count('0-1') + cmd:option('-e --environment', "Deployment to start. Can also be a path to a Lua file.", resty_env.value('APICAST_ENVIRONMENT')) + :count('*'):init(loaded_env):action('concat'):convert(load_env) + cmd:flag('--development --dev', 'Start in development environment'):action(function(arg, name) + insert(arg.environment, name) + end) - cmd:flag("-m --master", "Test the nginx config"):args('?') + cmd:flag("-m --master", "Control nginx master process.", 'on'):args('?') cmd:flag("-t --test", "Test the nginx config") cmd:flag("--debug", "Debug mode. Prints more information.") cmd:option("-c --configuration", @@ -183,17 +205,26 @@ local function configure(cmd) "Number of worker processes to start.", resty_env.value('APICAST_WORKERS') or Environment.default_config.worker_processes) cmd:option("-p --pid", "Path to the PID file.") - cmd:mutex( - cmd:flag('-b --boot', - "Load configuration on boot.", - resty_env.value('APICAST_CONFIGURATION_LOADER') == 'boot'), - cmd:flag('-l --lazy', - "Load configuration on demand.", - resty_env.value('APICAST_CONFIGURATION_LOADER') == 'lazy') - ) + + do + local target = 'configuration_loader' + local configuration_loader = resty_env.value('APICAST_CONFIGURATION_LOADER') + local function set_configuration_loader(value) + return function(args) args[target] = value end + end + + cmd:mutex( + cmd:flag('-b --boot', + "Load configuration on boot.", + configuration_loader == 'boot'):action(set_configuration_loader('boot')):target('configuration_loader'):init(configuration_loader), + cmd:flag('-l --lazy', + "Load configuration on demand.", + configuration_loader == 'lazy'):action(set_configuration_loader('lazy')):target('configuration_loader'):init(configuration_loader) + ) + end cmd:option("-i --refresh-interval", "Cache configuration for N seconds. Using 0 will reload on every request (not for production).", - resty_env.value('APICAST_CONFIGURATION_CACHE')) + resty_env.value('APICAST_CONFIGURATION_CACHE')):convert(tonumber) cmd:option("--policy-load-path", "Load path where to find policies. Entries separated by `:`.", diff --git a/gateway/src/apicast/cli/environment.lua b/gateway/src/apicast/cli/environment.lua index 2bd69ec67..a5aa4646d 100644 --- a/gateway/src/apicast/cli/environment.lua +++ b/gateway/src/apicast/cli/environment.lua @@ -75,18 +75,23 @@ _M.default_config = { local mt = { __index = _M } +--- Return loaded environments defined as environment variable. +-- @treturn {string,...} +function _M.loaded() + local value = resty_env.value('APICAST_LOADED_ENVIRONMENTS') + return re.split(value or '', [[\|]], 'jo') +end + --- Load an environment from files in ENV. -- @treturn Environment function _M.load() - local value = resty_env.value('APICAST_LOADED_ENVIRONMENTS') local env = _M.new() + local environments = _M.loaded() - if not value then + if not environments then return env end - local environments = re.split(value, '\\|', 'jo') - for i=1,#environments do assert(env:add(environments[i])) end @@ -143,7 +148,7 @@ function _M:add(env) local config = loadfile(path, 't', { print = print, inspect = require('inspect'), context = self._context, - tonumber = tonumber, tostring = tostring, + tonumber = tonumber, tostring = tostring, os = { getenv = resty_env.value }, pcall = pcall, require = require, assert = assert, error = error, }) diff --git a/gateway/src/apicast/configuration_loader.lua b/gateway/src/apicast/configuration_loader.lua index fc2a4aa1d..71dd1fea1 100644 --- a/gateway/src/apicast/configuration_loader.lua +++ b/gateway/src/apicast/configuration_loader.lua @@ -122,7 +122,7 @@ function boot.init(configuration) if config and init then ngx.log(ngx.DEBUG, 'downloaded configuration: ', config) else - ngx.log(ngx.EMERG, 'failed to load configuration, exiting (code ', code, ')\n', err) + ngx.log(ngx.EMERG, 'failed to load configuration, exiting (code ', code, ')\n', err or ngx.config.debug and debug.traceback()) os.exit(1) end @@ -213,8 +213,10 @@ function lazy.rewrite(configuration, host) return configuration end +local test = { init = noop, init_worker = noop, rewrite = noop } + local modes = { - boot = boot, lazy = lazy, default = 'lazy' + boot = boot, lazy = lazy, default = 'lazy', test = test } function _M.new(mode) diff --git a/t/configuration-loading-boot-with-config.t b/t/configuration-loading-boot-with-config.t index ec2a8ba20..fb07ae1fe 100644 --- a/t/configuration-loading-boot-with-config.t +++ b/t/configuration-loading-boot-with-config.t @@ -1,10 +1,8 @@ use lib 't'; use Test::APIcast::Blackbox 'no_plan'; -$ENV{APICAST_CONFIGURATION_LOADER} = 'boot'; - -env_to_nginx( - 'APICAST_CONFIGURATION_LOADER', +env_to_apicast( + 'APICAST_CONFIGURATION_LOADER' => 'boot' ); log_level('warn');