Policy: Camel proxy

Camel netty proxy was used via http-proxy policy, but on TLS
connections, the CONNECT method[0] is not allowed, so the request needs
to be terminated by camel-proxy and send to the upstream API.

With this change, we duplicate the http-proxy and we set the proxy to
terminate the connection so that will work correctly on camel TLS
connections.

[0] https://www.ietf.org/rfc/rfc2817.txt
Fix THREESCALE-4867

Signed-off-by: Eloy Coto <eloy.coto@acalustra.com>

CHANGELOG.md

@@ -25,6 +25,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 - Added custom_metrics policy [PR #1188](https://github.com/3scale/APIcast/pull/1188) [THREESCALE-5098](https://issues.jboss.org/browse/THREESCALE-5098)
 - New apicast_status Prometheus metric [THREESCALE-5417](https://issues.jboss.org/browse/THREESCALE-5417) [PR #1200](https://github.com/3scale/APIcast/pull/1200)
 
+- Added Camel policy [PR #1193](https://github.com/3scale/APIcast/pull/1193) [THREESCALE-4867](https://issues.jboss.org/browse/THREESCALE-4867)
 
 ## [3.8.0] - 2020-03-24
 

gateway/src/apicast/http_proxy.lua

@@ -84,7 +84,7 @@ local function current_path(uri)
     return format('%s%s%s', uri.path or ngx.var.uri, ngx.var.is_args, ngx.var.query_string or '')
 end
 
-local function forward_https_request(proxy_uri, uri)
+local function forward_https_request(proxy_uri, uri, skip_https_connect)
     -- This is needed to call ngx.req.get_body_data() below.
     ngx.req.read_body()
 
@@ -121,7 +121,7 @@ local function forward_https_request(proxy_uri, uri)
         end
     end
 
-    local httpc, err = http_proxy.new(request)
+    local httpc, err = http_proxy.new(request, skip_https_connect)
 
     if not httpc then
         ngx.log(ngx.ERR, 'could not connect to proxy: ',  proxy_uri, ' err: ', err)
@@ -172,7 +172,7 @@ function _M.request(upstream, proxy_uri)
         return
     elseif uri.scheme == 'https' then
         upstream:rewrite_request()
-        forward_https_request(proxy_uri, uri)
+        forward_https_request(proxy_uri, uri, upstream.skip_https_connect)
         return ngx.exit(ngx.OK) -- terminate phase
     else
         ngx.log(ngx.ERR, 'could not connect to proxy: ',  proxy_uri, ' err: ', 'invalid request scheme')

gateway/src/apicast/policy/camel/Readme.md

@@ -0,0 +1,77 @@
+#  Camel proxy policy
+
+This policy allows users to define a camel proxy where the traffic will be send
+over the defined proxy, the example traffic flow is the following:
+
+```
+   ,-.
+   `-'
+   /|\
+    |            ,-------.          ,---------.          ,----------.
+   / \           |Apicast|          |  CAMEL  |          |APIBackend|
+  User           `---+---'          `----+----'          `----------'
+   |  GET /resource  |                   |                    |
+   | --------------->|                   |                    |
+   |                 |                   |                    |
+   |                 |  Get /resource    |                    |
+   |                 |------------------>|                    |
+   |                 |                   |                    |
+   |                 |                   |  Get /resource/    |
+   |                 |                   | - - - - - - - - - >|
+   |                 |                   |                    |
+   |                 |                   |     response       |
+   |                 |                   |<- - - - - - - - - -|
+   |                 |                   |                    |
+   |                 |     response      |                    |
+   |                 |<------------------|                    |
+   |                 |                   |                    |
+   |                 |                   |                    |
+   | <---------------|                   |                    |
+  User           ,---+---.          ,----+----.          ,----------.
+   ,-.           |Apicast|          |  CAMEL  |          |APIBackend|
+   `-'           `-------'          `---------'          `----------'
+   /|\
+    |
+   / \
+```
+
+
+## Configuration
+
+```
+"policy_chain": [
+    {
+      "name": "apicast.policy.apicast"
+    },
+    {
+      "name": "apicast.policy.camel",
+      "configuration": {
+          "all_proxy": "http://192.168.15.103:8888/",
+          "https_proxy": "https://192.168.15.103:8888/",
+          "http_proxy": "https://192.168.15.103:8888/"
+      }
+    }
+]
+```
+
+- If http_proxy or https_proxy is not defined the all_proxy will be taken. 
+
+## Caveats
+
+- This policy will disable all load-balancing policies and traffic will be
+  always send to the proxy. 
+- In case of HTTP_PROXY, HTTPS_PROXY or ALL_PROXY parameters are defined, this
+  policy will overwrite those values. 
+- Proxy connection does not support authentication, if you need auth, please use
+  headers policy.
+
+
+## Example Use case
+
+This policy was designed to be able to apply more fined grained policies and
+transformation using Apache Camel.
+
+An example project can be found
+[here](https://github.com/zregvart/camel-netty-proxy). This project is an HTTP
+Proxy that transforms to uppercase all the response body given by the API
+backend.

gateway/src/apicast/policy/camel/apicast-policy.json

@@ -0,0 +1,27 @@
+{
+  "$schema": "http://apicast.io/policy-v1/schema#manifest#",
+  "name": "Camel Service",
+  "summary": "Adds an Camel proxy to the service.",
+  "description": [
+    "With this policy all the traffic for this service will be routed accross ",
+    "the defined proxy"
+  ],
+  "version": "builtin",
+  "configuration": {
+      "type": "object",
+      "properties": {
+        "all_proxy": {
+          "description": "Defines a HTTP proxy to be used for connecting to services if a protocol-specific proxy is not specified. Authentication is not supported.",
+          "type": "string"
+        },
+        "https_proxy": {
+          "description": "Defines a HTTPS proxy to be used for connecting to HTTPS services. Authentication is not supported",
+          "type": "string"
+        },
+        "http_proxy": {
+          "description": "Defines a HTTP proxy to be used for connecting to HTTP services. Authentication is not supported",
+          "type": "string"
+        }
+      }
+  }
+}

gateway/src/apicast/policy/camel/camel.lua

@@ -0,0 +1,59 @@
+local policy = require('apicast.policy')
+local _M = policy.new('http_proxy', 'builtin')
+
+local resty_url = require 'resty.url'
+local ipairs = ipairs
+
+local new = _M.new
+
+local proxies = {"http", "https"}
+
+function _M.new(config)
+  local self = new(config)
+  self.proxies = {}
+
+  if config.all_proxy then
+    local err
+    self.all_proxy, err =  resty_url.parse(config.all_proxy)
+    if err then
+      ngx.log(ngx.WARN, "All proxy '", config.all_proxy, "' is not correctly defined, err:", err)
+    end
+  end
+
+  for _, proto in ipairs(proxies) do
+    local val, err =  resty_url.parse(config[string.format("%s_proxy", proto)])
+    if err then
+      ngx.log(ngx.WARN, proto, " proxy is not correctly defined, err: ", err)
+    end
+    self.proxies[proto] = val or self.all_proxy
+  end
+  return self
+end
+
+local function find_proxy(self, scheme)
+  return self.proxies[scheme]
+end
+
+function _M:access(context)
+  local upstream = context.get_upstream()
+  if not upstream then
+    return
+  end
+
+  upstream:set_skip_https_connect_on_proxy()
+end
+
+function _M:export()
+  -- This get_http_proxy function will be called in upstream just in case if a
+  -- proxy is defined.
+  return  {
+    get_http_proxy = function(uri)
+      if not uri.scheme then
+        return nil
+      end
+      return find_proxy(self, uri.scheme)
+    end
+  }
+end
+
+return _M

gateway/src/apicast/policy/camel/init.lua

@@ -0,0 +1 @@
+return require("camel")

gateway/src/apicast/upstream.lua

@@ -194,6 +194,9 @@ function _M:set_host_header()
     return host, nil
 end
 
+function _M:set_skip_https_connect_on_proxy()
+  self.skip_https_connect = true
+end
 --- Execute the upstream.
 --- @tparam table context any table (policy context, ngx.ctx) to store the upstream for later use by balancer
 function _M:call(context)

gateway/src/resty/http/proxy.lua

@@ -37,6 +37,16 @@ local function connect_direct(httpc, request)
     return httpc
 end
 
+local function _connect_tls_direct(httpc, request, host, port)
+
+    local uri = request.uri
+
+    local ok, err = httpc:ssl_handshake(nil, uri.host, request.ssl_verify)
+    if not ok then return nil, err end
+
+    return httpc
+end
+
 local function _connect_proxy_https(httpc, request, host, port)
     -- When the connection is reused the tunnel is already established, so
     -- the second CONNECT request would reach the upstream instead of the proxy.
@@ -61,7 +71,7 @@ local function _connect_proxy_https(httpc, request, host, port)
     return httpc
 end
 
-local function connect_proxy(httpc, request)
+local function connect_proxy(httpc, request, skip_https_connect)
     local uri = request.uri
     local host, port = httpc:resolve(uri.host, uri.port, uri)
     local proxy_uri = request.proxy
@@ -88,9 +98,10 @@ local function connect_proxy(httpc, request)
         -- http proxy needs absolute URL as the request path
         request.path = format('%s://%s:%s%s', uri.scheme, host, port, uri.path or '/')
         return httpc
-
+    elseif uri.scheme == 'https' and skip_https_connect then
+        request.path = format('%s://%s:%s%s', uri.scheme, host, port, request.path or '/')
+        return _connect_tls_direct(httpc, request, host, port)
     elseif uri.scheme == 'https' then
-
         return _connect_proxy_https(httpc, request, host, port)
 
     else
@@ -113,15 +124,15 @@ local function find_proxy_url(request)
     return request.proxy_uri or _M.find(uri)
 end
 
-local function connect(request)
+local function connect(request, skip_https_connect)
     local httpc = http.new()
     local proxy_uri = find_proxy_url(request)
 
     request.ssl_verify = request.options and request.options.ssl and request.options.ssl.verify
     request.proxy = proxy_uri
 
     if proxy_uri then
-        return connect_proxy(httpc, request)
+        return connect_proxy(httpc, request, skip_https_connect)
     else
         return connect_direct(httpc, request)
     end

spec/policy/camel/camel_spec.lua

@@ -0,0 +1,61 @@
+local camel_policy = require('apicast.policy.camel')
+local resty_url = require 'resty.url'
+
+describe('Camel policy', function()
+  local all_proxy_val = "http://all.com"
+  local http_proxy_val = "http://plain.com"
+  local https_proxy_val = "http://secure.com"
+
+  local http_uri = {scheme="http"}
+  local https_uri = {scheme="https"}
+
+  it("http[s] proxies are defined if all_proxy is in there", function()
+    local proxy = camel_policy.new({
+      all_proxy = all_proxy_val
+    })
+    local callback = proxy:export()
+
+    assert.same(callback.get_http_proxy(http_uri), resty_url.parse(all_proxy_val))
+    assert.same(callback.get_http_proxy(https_uri), resty_url.parse(all_proxy_val))
+  end)
+
+  it("all_proxy does not overwrite http/https proxies", function()
+    local proxy = camel_policy.new({
+      all_proxy = all_proxy_val,
+      http_proxy = http_proxy_val,
+      https_proxy = https_proxy_val
+    })
+    local callback = proxy:export()
+
+    assert.same(callback.get_http_proxy(http_uri), resty_url.parse(http_proxy_val))
+    assert.same(callback.get_http_proxy(https_uri), resty_url.parse(https_proxy_val))
+  end)
+
+  it("empty config return all nil", function()
+    local proxy = camel_policy.new({})
+    local callback = proxy:export()
+
+    assert.is_nil(callback.get_http_proxy(https_uri))
+    assert.is_nil(callback.get_http_proxy(http_uri))
+  end)
+
+  describe("get_http_proxy callback", function()
+    local callback = camel_policy.new({
+        all_proxy = all_proxy_val
+    }):export()
+
+    it("Valid protocol", function()
+
+      local result = callback.get_http_proxy(
+        resty_url.parse("http://google.com"))
+      assert.same(result, resty_url.parse(all_proxy_val))
+    end)
+
+    it("invalid protocol", function()
+      local result = callback:get_http_proxy(
+        {}, {scheme="invalid"})
+      assert.is_nil(result)
+    end)
+
+  end)
+end)