From d880c80afb63a124ba9dd8d8dc3e0ddbf7441bfd Mon Sep 17 00:00:00 2001 From: Eloy Coto Date: Thu, 25 Apr 2019 19:07:38 +0200 Subject: [PATCH 1/3] [Test] Migrate test to Blackbox schema Migrated t/apicast-subset-of-services.t tests to the new testing Blackbox schema Signed-off-by: Eloy Coto --- t/apicast-subset-of-services.t | 75 +++++++++++++++++----------------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/t/apicast-subset-of-services.t b/t/apicast-subset-of-services.t index eb4b1ef20..2b57f8390 100644 --- a/t/apicast-subset-of-services.t +++ b/t/apicast-subset-of-services.t @@ -1,55 +1,56 @@ -use lib 't'; -use Test::APIcast 'no_plan'; +use Test::APIcast::Blackbox 'no_plan'; run_tests(); __DATA__ === TEST 1: multi service configuration limited to specific service ---- main_config -env APICAST_SERVICES_LIST=42,21; ---- http_config - include $TEST_NGINX_UPSTREAM_CONFIG; - lua_package_path "$TEST_NGINX_LUA_PATH"; - init_by_lua_block { - require('apicast.configuration_loader').mock({ - services = { - { - id = 42, - backend_version = 1, - proxy = { - api_backend = "http://127.0.0.1:$TEST_NGINX_SERVER_PORT/api-backend/one/", - hosts = { 'one' }, - proxy_rules = { - { pattern = '/', http_method = 'GET', metric_system_name = 'one', delta = 1 } - } +--- env eval +("APICAST_SERVICES_LIST", "42,21") +--- configuration +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "hosts": [ + "one" + ], + "api_backend": "http://test:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { + "http_method": "GET", + "delta": 1, + "metric_system_name": "one", + "pattern": "/" } - }, - { - id = 11, - proxy = { - hosts = { 'two' } - } - } - } - }) - } - lua_shared_dict api_keys 10m; ---- config - include $TEST_NGINX_APICAST_CONFIG; - + ] + }, + "id": 42 + }, + { + "proxy": { + "hosts": [ + "two" + ] + }, + "id": 11 + } + ] +} +--- backend location /transactions/authrep.xml { content_by_lua_block { ngx.exit(200) } } - - location ~ /api-backend(/.+) { - echo 'yay, api backend: $1'; +--- upstream + location ~ / { + echo 'yay, api backend'; } --- pipelined_requests eval ["GET /?user_key=1","GET /?user_key=2"] --- more_headers eval ["Host: one", "Host: two"] --- response_body eval -["yay, api backend: /one/\n", ""] +["yay, api backend\n", ""] --- error_code eval [200, 404] From b9e740f0beb640481f17b04c4377c794af68577a Mon Sep 17 00:00:00 2001 From: Eloy Coto Date: Thu, 25 Apr 2019 14:42:41 +0200 Subject: [PATCH 2/3] [THREESCALE-1524][config] add support to filter services by public URL. This commit adds the option to filter services based on the endpoint. This commit expose a new config flag using the env variable `APICAST_SERVICES_FILTER_BY_URL`, services filter happens on `configuration.filter_services`. Signed-off-by: Eloy Coto --- CHANGELOG.md | 1 + doc/parameters.md | 23 +++++++ gateway/src/apicast/configuration.lua | 32 +++++++--- gateway/src/apicast/configuration/service.lua | 20 +++++++ spec/configuration_spec.lua | 60 +++++++++++++++++-- 5 files changed, 123 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 031c048a0..5ccd0e6b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added - Ability to configure client certificate chain depth [PR #1006](https://github.com/3scale/APIcast/pull/1006) +- You can filter services by endpoint name using Regexp [PR #1022](https://github.com/3scale/APIcast/pull/1022) [THREESCALE-1524](https://issues.jboss.org/browse/THREESCALE-1524) ### Fixed diff --git a/doc/parameters.md b/doc/parameters.md index f16f7cbd2..b83c77f2f 100644 --- a/doc/parameters.md +++ b/doc/parameters.md @@ -186,6 +186,29 @@ before the client is throttled by adding latency. When set to _true_, APIcast will log the response code of the response returned by the API backend in 3scale. In some plans this information can later be consulted from the 3scale admin portal. Find more information about the Response Codes feature on the [3scale support site](https://access.redhat.com/documentation/en-us/red_hat_3scale/2.saas/html/analytics/response-codes-tracking). +### `APICAST_SERVICES_FILTER_BY_URL` +**Value:** a PCRE (Perl Compatible Regular Expression) +**Example:** .*.example.com + +Used to filter the service configured in the 3scale API Manager, the filter +matches with the public base URL. Services that do not match the filter will be +discarded. If the regular expression cannot be compiled no services will be +loaded. + +Note: If a service does not match, but is included in the +`APICAST_SERVICES_LIST`, service will not be discarded + +Example: + +Regexp Filter: http:\/\/.*.google.com +Service 1: backend endpoint http://www.google.com +Service 2: backend endpoint http://www.yahoo.com +Service 3: backend endpoint http://mail.google.com +Service 4: backend endpoint http://mail.yahoo.com + +The services that will be configured in Apicast will be 1 and 3. Services 2 and +4 will be discarded. + ### `APICAST_SERVICES_LIST` **Value:** a comma-separated list of service IDs diff --git a/gateway/src/apicast/configuration.lua b/gateway/src/apicast/configuration.lua index 90a4a3f3f..359bc051c 100644 --- a/gateway/src/apicast/configuration.lua +++ b/gateway/src/apicast/configuration.lua @@ -12,7 +12,6 @@ local insert = table.insert local setmetatable = setmetatable local null = ngx.null -local re = require 'ngx.re' local env = require 'resty.env' local resty_url = require 'resty.url' local util = require 'apicast.util' @@ -20,6 +19,9 @@ local policy_chain = require 'apicast.policy_chain' local mapping_rule = require 'apicast.mapping_rule' local tab_new = require('resty.core.base').new_tab +local re = require 'ngx.re' +local match = ngx.re.match + local mt = { __index = _M, __tostring = function() return 'Configuration' end } local function map(func, tbl) @@ -83,7 +85,6 @@ function _M.parse_service(service) local proxy = service.proxy or empty_t local backend = backend_endpoint(proxy) - return Service.new({ id = tostring(service.id or 'default'), backend_version = backend_version, @@ -140,21 +141,34 @@ function _M.services_limit() end function _M.filter_services(services, subset) - subset = subset and util.to_hash(subset) or _M.services_limit() - if not subset or not next(subset) then return services end + local selected_services = {} + local service_regexp_filter = env.value("APICAST_SERVICES_FILTER_BY_URL") + + if service_regexp_filter then + -- Checking that the regexp sent is correct, if not an empty service list + -- will be returned. + local _, err = match("", service_regexp_filter, 'oj') + if err then + -- @todo this return and empty list, Apicast will continue running maybe + -- process need to be stopped here. + ngx.log(ngx.ERR, "APICAST_SERVICES_FILTER_BY_URL cannot compile and all services are filtering out, error: ", err) + return selected_services + end + end - local s = {} + subset = subset and util.to_hash(subset) or _M.services_limit() + if (not subset or not next(subset)) and not service_regexp_filter then return services end + subset = subset or {} for i = 1, #services do local service = services[i] - if subset[service.id] then - insert(s, service) + if service:match_host(service_regexp_filter) or subset[service.id] then + insert(selected_services, service) else ngx.log(ngx.WARN, 'filtering out service ', service.id) end end - - return s + return selected_services end function _M.new(configuration) diff --git a/gateway/src/apicast/configuration/service.lua b/gateway/src/apicast/configuration/service.lua index acfb04809..566abc0bb 100644 --- a/gateway/src/apicast/configuration/service.lua +++ b/gateway/src/apicast/configuration/service.lua @@ -8,7 +8,9 @@ local rawget = rawget local lower = string.lower local gsub = string.gsub local select = select + local re = require 'ngx.re' +local match = ngx.re.match local http_authorization = require 'resty.http_authorization' @@ -264,4 +266,22 @@ function _M:get_usage(method, path) end end +--- Validate that the given regexp match with one of the service hosts. +--- This function needs a valid regexp, if not will return false +-- @tparam string regexp Regular expresion to match with the host +-- @return bool true if match +function _M:match_host(regexp) + if not regexp or not self.hosts then + return false + end + + for j = 1, #self.hosts do + local val, _ = match(self.hosts[j], regexp, 'oj') + if val then + return true + end + end + return false +end + return _M diff --git a/spec/configuration_spec.lua b/spec/configuration_spec.lua index de8f639a6..cc10a6745 100644 --- a/spec/configuration_spec.lua +++ b/spec/configuration_spec.lua @@ -25,7 +25,6 @@ describe('Configuration object', function() assert.same('example.com', config.hostname_rewrite) end) - it('has a default message, content-type, and status for the auth failed error', function() local config = configuration.parse_service({}) @@ -112,19 +111,72 @@ describe('Configuration object', function() end) describe('.filter_services', function() + local Service = require 'apicast.configuration.service' local filter_services = configuration.filter_services it('works with nil', function() - local services = { { id = '42' } } + local services = { Service.new({id="42"})} assert.equal(services, filter_services(services)) end) it('works with table with ids', function() - local services = { { id = '42' } } - + local services = { Service.new({id="42"})} assert.same(services, filter_services(services, { '42' })) assert.same({}, filter_services(services, { '21' })) end) + + describe("with service filter", function() + + local mockservices = { + Service.new({id="42", hosts={"test.foo.com", "test.bar.com"}}), + Service.new({id="12", hosts={"staging.foo.com"}}), + Service.new({id="21", hosts={"prod.foo.com"}}), + } + + it("with empty env variable", function() + env.set('APICAST_SERVICES_FILTER_BY_URL', '') + assert.same(filter_services(mockservices, nil), mockservices) + end) + + it("it does not discard any service when there is not regex", function() + assert.same(filter_services(mockservices, nil), mockservices) + end) + + it("reads from environment variable", function() + env.set('APICAST_SERVICES_FILTER_BY_URL', '.*.foo.com') + assert.same(filter_services(mockservices, nil), mockservices) + + env.set('APICAST_SERVICES_FILTER_BY_URL', '^test.*') + assert.same(filter_services(mockservices, nil), {mockservices[1]}) + + env.set('APICAST_SERVICES_FILTER_BY_URL', '^(test|prod).*') + assert.same(filter_services(mockservices, nil), {mockservices[1], mockservices[3]}) + end) + + it("matches the second host", function() + env.set('APICAST_SERVICES_FILTER_BY_URL', '^test.bar.com') + assert.same(filter_services(mockservices, nil), {mockservices[1]}) + end) + + it("validates invalid regexp", function() + env.set('APICAST_SERVICES_FILTER_BY_URL', '^]') + assert.same(filter_services(mockservices, nil), {}) + end) + + it("combination with service list", function() + env.set('APICAST_SERVICES_FILTER_BY_URL', '^test.*') + env.set('APICAST_SERVICES_LIST', '42,21') + + assert.same(filter_services(mockservices, {"21"}), { + mockservices[1], + mockservices[3]}) + + assert.same(filter_services(mockservices, nil), { + mockservices[1], + mockservices[3]}) + end) + end) + end) insulate('.services_limit', function() From 6c7702bdf8b52a832d7ecbb275bd31efc002dbd1 Mon Sep 17 00:00:00 2001 From: Eloy Coto Date: Thu, 25 Apr 2019 19:21:04 +0200 Subject: [PATCH 3/3] Test: Added APICAST_SERVICE_FILTER_BY_URL integration test. This test validates that the parameter `APICAST_SERVICE_FILTER_BY_URL` is working correctly across multiple services and the service discarding is working correctly. Signed-off-by: Eloy Coto --- t/apicast-subset-of-services.t | 130 +++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/t/apicast-subset-of-services.t b/t/apicast-subset-of-services.t index 2b57f8390..0e4557d4a 100644 --- a/t/apicast-subset-of-services.t +++ b/t/apicast-subset-of-services.t @@ -54,3 +54,133 @@ __DATA__ ["yay, api backend\n", ""] --- error_code eval [200, 404] + + + +=== TEST 2: multi service configuration limited with Regexp Filter +--- env eval +("APICAST_SERVICES_FILTER_BY_URL", "^on*") +--- configuration +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "hosts": [ + "one" + ], + "api_backend": "http://test:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { + "http_method": "GET", + "delta": 1, + "metric_system_name": "one", + "pattern": "/" + } + ] + }, + "id": 42 + }, + { + "proxy": { + "hosts": [ + "two" + ] + }, + "id": 11 + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { ngx.exit(200) } + } +--- upstream + location ~ / { + echo 'yay, api backend'; + } +--- pipelined_requests eval +["GET /?user_key=1","GET /?user_key=2"] +--- more_headers eval +["Host: one", "Host: two"] +--- response_body eval +["yay, api backend\n", ""] +--- error_code eval +[200, 404] + +=== TEST 3: multi service configuration limited with Regexp Filter and service list +--- env eval +( +"APICAST_SERVICES_FILTER_BY_URL", "^on*", +"APICAST_SERVICES_LIST", "21" +) +--- configuration +{ + "services": [ + { + "backend_version": 1, + "proxy": { + "hosts": [ + "one" + ], + "api_backend": "http://test:$TEST_NGINX_SERVER_PORT/", + "proxy_rules": [ + { + "http_method": "GET", + "delta": 1, + "metric_system_name": "one", + "pattern": "/" + } + ] + }, + "id": 42 + }, + { + "backend_version": 1, + "proxy": { + "hosts": [ + "two" + ], + "api_backend": "http://test:$TEST_NGINX_SERVER_PORT/two", + "proxy_rules": [ + { + "http_method": "GET", + "delta": 1, + "metric_system_name": "one", + "pattern": "/" + } + ] + }, + "id": 21 + }, + { + "proxy": { + "hosts": [ + "three" + ] + }, + "id": 11 + } + ] +} +--- backend + location /transactions/authrep.xml { + content_by_lua_block { ngx.exit(200) } + } +--- upstream + location / { + echo 'yay, api backend'; + } + + location /two { + echo 'yay, api backend two'; + } + +--- pipelined_requests eval +["GET /?user_key=1","GET /?user_key=2", "GET /?user_key=3"] +--- more_headers eval +["Host: one", "Host: two", "Host: three"] +--- response_body eval +["yay, api backend\n", "yay, api backend two\n", ""] +--- error_code eval +[200, 200, 404]