Policy: logging add custom access log options

Some users requested different ways to log access log with more metadata,
different formats or conditional logging based on multiple request values.

This policy address this, two new variables are now set, where allow or disallow
to print a custom log message, and another one `extened_access_log` just store
all the information to print that.

Policy has multiple options, here a few examples:

Custom log format
```
{
  "name": "apicast.policy.logging",
  "configuration": {
    "enable_access_logs": false
    "custom_logging": "\"{{request}}\" to service {{service.id}} and {{service.name}}",
  }
}
```

Only log the entry if status is 200

```
{
  "name": "apicast.policy.logging",
  "configuration": {
    "enable_access_logs": false
    "custom_logging": "\"{{request}}\" to service {{service.id}} and {{service.name}}",
    "condition": {
      "operations": [
        {"op": "==", "match": "{{status}}", "match_type": "liquid", "value": "200"}
      ],
      "combine_op": "and"
    }
  }
}
```

This commit fixed 3scale#1082 and THREESCALE-1234 and THREESCALE-2876

Signed-off-by: Eloy Coto <eloy.coto@gmail.com>

gateway/http.d/apicast.conf.liquid

@@ -1,5 +1,11 @@
 log_format time '[$time_local] $host:$server_port $remote_addr:$remote_port "$request" $status $body_bytes_sent ($request_time) $post_action_impact';
 
+map $status $extended_access_log {
+    default '';
+}
+
+log_format extended escape=none '$extended_access_log';
+
 server {
   listen {{ port.management | default: 8090 }};
   server_name {{ server_name.management | default: 'management _' }};
@@ -45,6 +51,8 @@ server {
 
   set $access_logs_enabled '1';
   access_log {{ access_log_file | default: "/dev/stdout" }} time if=$access_logs_enabled;
+  set $extended_access_logs_enabled '0';
+  access_log {{ access_log_file | default: "/dev/stdout" }} extended if=$extended_access_logs_enabled;
 
   {%- assign http_port = port.apicast | default: 8080 %}
   {%- assign https_port = env.APICAST_HTTPS_PORT %}

gateway/src/apicast/policy/logging/logging.lua

@@ -1,13 +1,22 @@
 --- Logging policy
 
 local _M  = require('apicast.policy').new('Logging Policy', 'builtin')
-
 local new = _M.new
 
-local default_enable_access_logs = true
+local Condition = require('apicast.conditions.condition')
+local LinkedList = require('apicast.linked_list')
+local Operation = require('apicast.conditions.operation')
+local TemplateString = require('apicast.template_string')
+local cjson = require('cjson')
 
 -- Defined in ngx.conf.liquid and used in the 'access_logs' directive.
 local ngx_var_access_logs_enabled = 'access_logs_enabled'
+local ngx_var_extended_access_logs_enabled = 'extended_access_logs_enabled'
+local ngx_var_extended_access_log = 'extended_access_log'
+
+local default_enable_access_logs = true
+local default_template_type = 'plain'
+local default_combine_op = "and"
 
 -- Returns the value for the ngx var above from a boolean that indicates
 -- whether access logs are enabled or not.
@@ -29,12 +38,101 @@ function _M.new(config)
   end
 
   self.enable_access_logs_val = val_for_ngx_var[enable_access_logs]
+  self.custom_logging = config.custom_logging
+  self.enable_json_logs = config.enable_json_logs
+	self.json_object_config = config.json_object_config or {}
+
+  if config.condition then
+    ngx.log(ngx.DEBUG, 'Enabling extended log with conditions')
+    local operations = {}
+    for _, operation in ipairs(config.condition.operations) do
+      table.insert( operations,
+        Operation.new(
+          operation.match, operation.match_type,
+          operation.op,
+          operation.value, operation.value_type or default_template_type))
+    end
+    self.condition = Condition.new( operations, config.condition.combine_op or default_combine_op)
+  end
 
   return self
 end
 
-function _M:log()
+local function get_request_context(context)
+  local ctx = { }
+  ctx.req = {
+    headers=ngx.req.get_headers(),
+  }
+
+  ctx.resp = {
+    headers=ngx.resp.get_headers(),
+  }
+
+  ctx.service = context.service or {}
+  return LinkedList.readonly(ctx, ngx.var)
+end
+
+--- log_dump_json: returns an string with the json output.
+function _M:log_dump_json(extended_context)
+  local result = {}
+  for _, value in ipairs(self.json_object_config) do
+    result[value.key] = TemplateString.new(value.value, value.value_type or default_template_type):render(extended_context)
+  end
+
+	local status, data = pcall(cjson.encode, result)
+	if not status then
+		ngx.log(ngx.WARN, "cannot serialize json on logging, err:", data)
+		-- Disable access log due to no valid information can be returned
+		self:disable_extended_access_log()
+		return ""
+	end
+
+	return data
+end
+
+-- log_dump_line: render the liquid custom_logging value and return it.
+function _M:log_dump_line(extended_context)
+  local tmpl = TemplateString.new(self.custom_logging, "liquid")
+  return tmpl:render(extended_context)
+end
+
+-- get_log_line return the log line based on the kind of log defined in the
+-- service, if Json is enabled will dump a json object, if not will render the
+-- simple log line.
+function _M:get_log_line(extended_context)
+  if self.enable_json_logs then
+    return self:log_dump_json(extended_context)
+  end
+  return self:log_dump_line(extended_context)
+end
+
+function _M.enable_extended_access_log()
+	ngx.var[ngx_var_extended_access_logs_enabled] = 1
+end
+
+function _M.disable_extended_access_log()
+	ngx.var[ngx_var_extended_access_logs_enabled] = 0
+end
+
+function _M:log(context)
   ngx.var[ngx_var_access_logs_enabled] = self.enable_access_logs_val
+  if not (self.custom_logging or self.enable_json_logs) then
+    return
+  end
+
+  -- Extended log is now enaled, disable the default access_log
+  ngx.var[ngx_var_access_logs_enabled] = 0
+
+  local extended_context = get_request_context(context or {})
+  if self.condition and not self.condition:evaluate(extended_context) then
+    -- Access log is disabled here, request does not match, so log is disabled
+    -- for this request
+		self:disable_extended_access_log()
+    return
+  end
+
+	self:enable_extended_access_log()
+  ngx.var[ngx_var_extended_access_log] = self:get_log_line(extended_context)
 end
 
 return _M