THREESCALE-8373 Pagination services and proxy config endpoints

CHANGELOG.md

@@ -11,6 +11,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 - Opentelemetry support. Opentracing is now deprecated [PR #1379](https://github.com/3scale/APIcast/pull/1379) [THREESCALE-7735](https://issues.redhat.com/browse/THREESCALE-7735)
 - `/admin/api/account/proxy_configs` endpoint for configuration loading [PR #1352](https://github.com/3scale/APIcast/pull/1352) [THREESCALE-8508](https://issues.redhat.com/browse/THREESCALE-8508)
+- Pagination of services and proxy config endpoints [PR #1397](https://github.com/3scale/APIcast/pull/1397) [THREESCALE-8373](https://issues.redhat.com/browse/THREESCALE-8373)
 
 ### Removed
 

doc/parameters.md

@@ -46,6 +46,7 @@ Defines how to load the configuration.
 In `boot` mode APIcast will request the configuration to the API manager when the gateway starts.
 In `lazy` mode APIcast will load the configuration on demand for each incoming request (to guarantee a complete refresh on each request `APICAST_CONFIGURATION_CACHE` should be set to `0`).
 
+
 ### `APICAST_CUSTOM_CONFIG`
 
 **Deprecated:** Use [policies](./policies.md) instead.
@@ -241,6 +242,8 @@ Replace `${ID}` with the actual Service ID. The value should be the configuratio
 
 Setting it to a particular version will prevent it from auto-updating and will always use that version.
 
+**Note**: This env var cannot be used with `THREESCALE_PORTAL_ENDPOINT` pointing to custom path (i.e. master path).
+
 ### `APICAST_UPSTREAM_RETRY_CASES`
 
 **Default**:
@@ -320,6 +323,22 @@ The value will also be used in the header `X-3scale-User-Agent` in the authorize
 
 URI that includes your password and portal endpoint in the following format: `<schema>://<password>@<admin-portal-domain>`. The `<password>` can be either the provider key or an access token for the 3scale Account Management API. `<admin-portal-domain>` is the URL used to log into the admin portal.
 
+The path appended to `THREESCALE_PORTAL_ENDPOINT` is:
+
+|                      | `APICAST_CONFIGURATION_LOADER`=boot                            | `APICAST_CONFIGURATION_LOADER`=lazy                                     |
+|----------------------|----------------------------------------------------------------|-------------------------------------------------------------------------|
+| endpoint has no path | `/admin/api/account/proxy_configs/${env}.json?version=version&page=X&per_page=500` | `/admin/api/account/proxy_configs/${env}.json?host=host&version=version&page=X&per_page=500` |
+| endpoint has a path  | `/${env}.json`                                                 | `/${env}.json?host=host`                                                 |
+
+The exception to the logic in table above would be when the env var `APICAST_SERVICE_%s_CONFIGURATION_VERSION` is provided.
+In that case, the gateway would load service's proxy configuration one by one:
+* 1 request to `/admin/api/services.json?page=X&per_page=500` (which is paginated and the gateway will iterate over the pages)
+* N requests to `/admin/api/services/${SERVICE_ID}/proxy/configs/${ENVIRONMENT}/{VERSION}.json`.
+
+Note that when the `THREESCALE_PORTAL_ENDPOINT` has no path, the gateway will iterate over the pages of `/admin/api/account/proxy_configs/${env}.json` sending `pages` and `per_page` query parameters.
+
+**Note**: Pages in 3scale API services and proxy config endpoints were implemented on 3scale 2.10 [THREESCALE-4528](https://issues.redhat.com/browse/THREESCALE-4528). Older releases should not be used.
+
 **Example:** `https://access-token@account-admin.3scale.net`.
 
 When `THREESCALE_PORTAL_ENDPOINT` environment variable is provided, the gateway will download the configuration from 3scale on initializing. The configuration includes all the settings provided on the Integration page of the API(s).

gateway/src/apicast/configuration_loader/remote_v2.lua

@@ -4,7 +4,6 @@ local len = string.len
 local ipairs = ipairs
 local insert = table.insert
 local rawset = rawset
-local encode_args = ngx.encode_args
 local tonumber = tonumber
 local pcall = pcall
 
@@ -88,6 +87,10 @@ local function services_index_endpoint(portal_endpoint)
   return resty_url.join(portal_endpoint, '/admin/api/services.json')
 end
 
+local function proxy_configs_index_endpoint(portal_endpoint, env)
+  return resty_url.join(portal_endpoint, '/admin/api/account/proxy_configs/'..env..'.json')
+end
+
 local function service_config_endpoint(portal_endpoint, service_id, env, version)
   local version_override = resty_env.get(
       format('APICAST_SERVICE_%s_CONFIGURATION_VERSION', service_id)
@@ -100,15 +103,9 @@ local function service_config_endpoint(portal_endpoint, service_id, env, version
   )
 end
 
-local function parse_resp_body(self, resp_body)
-  local ok, res = pcall(cjson.decode, resp_body)
-  if not ok then return nil, res end
-  local json = res
-
+local function parse_proxy_configs(self, proxy_configs)
   local config = { services = array(), oidc = array() }
 
-  local proxy_configs = json.proxy_configs or {}
-
   for i, proxy_conf in ipairs(proxy_configs) do
     local proxy_config = proxy_conf.proxy_config
 
@@ -137,20 +134,14 @@ local function parse_resp_body(self, resp_body)
   return cjson.encode(config)
 end
 
-local function is_service_filter_by_url_set()
-  if resty_env.value('APICAST_SERVICES_FILTER_BY_URL') then
-    return true
-  else
-    return false
-  end
-end
+local function parse_resp_body(self, resp_body)
+  local ok, res = pcall(cjson.decode, resp_body)
+  if not ok then return nil, res end
+  local json = res
 
-local function is_service_list_set()
-  if resty_env.value('APICAST_SERVICES_LIST') then
-    return true
-  else
-    return false
-  end
+  local proxy_configs = json.proxy_configs or {}
+
+  return parse_proxy_configs(self, proxy_configs)
 end
 
 local function is_service_version_set()
@@ -163,14 +154,6 @@ local function is_service_version_set()
   return false
 end
 
--- Returns a table that represents paths and query parameters for the current endpoint:
--- http://${THREESCALE_PORTAL_ENDPOINT}/<env>.json?host=host
--- http://${THREESCALE_PORTAL_ENDPOINT}/admin/api/account/proxy_configs/<env>.json?host=host&version=version
-local function configuration_endpoint_params(env, host, portal_endpoint_path)
-  return portal_endpoint_path and {path = env, args = {host = host}}
-    or {path = '/admin/api/account/proxy_configs/' .. env, args = {host = host, version = "latest"} }
-end
-
 function _M:index_per_service()
   local http_client = self.http_client
 
@@ -226,7 +209,7 @@ function _M:index_per_service()
   return cjson.encode(configs)
 end
 
-function _M:index(host)
+function _M:index_custom_path(host)
   local http_client = self.http_client
 
   if not http_client then
@@ -240,9 +223,10 @@ function _M:index(host)
     return nil, 'missing environment'
   end
 
-  local endpoint_params = configuration_endpoint_params(env, host, proxy_config_path)
-  local base_url = resty_url.join(self.endpoint, endpoint_params.path .. '.json')
-  local query_args = encode_args(endpoint_params.args) ~= '' and '?'..encode_args(endpoint_params.args)
+  -- http://${THREESCALE_PORTAL_ENDPOINT}/<env>.json?host=host
+  local base_url = resty_url.join(self.endpoint, env..'.json')
+  local encoded_args = ngx.encode_args({host = host})
+  local query_args = encoded_args ~= '' and '?'..encoded_args
   local url = query_args and base_url..query_args or base_url
 
   local res, err = http_client.get(url)
@@ -259,6 +243,83 @@ function _M:index(host)
   return nil, 'invalid status'
 end
 
+-- Returns existing proxy configs in a single page
+-- @param http_client the http client object
+-- @param portal_endpoint 3scale API endpoint
+-- @param host proxy config filter based on request hostname. Optional, can be nil.
+-- @param env gateway environment
+-- @param page page in the paginated list. Defaults to 1 for the API, as the client will not send the page param.
+-- @param per_page number of results per page. Default and max is 500 for the API, as the client will not send the per_page param.
+local proxy_configs_per_page = function(http_client, portal_endpoint, host, env, page, per_page)
+  local args = { host = host, version = "latest", page = page, per_page = per_page }
+
+  local query_args = '?'..ngx.encode_args(args)
+  local base_url = proxy_configs_index_endpoint(portal_endpoint, env)
+  local url = base_url..query_args
+
+  -- http://${THREESCALE_PORTAL_ENDPOINT}/admin/api/account/proxy_configs/<env>.json?host=host&version=latest&page=1&per_page=500
+  local res, err = http_client.get(url)
+
+  if not res and err then
+    ngx.log(ngx.DEBUG, 'proxy configs get error: ', err, ' url: ', url)
+    return nil, err
+  end
+
+  ngx.log(ngx.DEBUG, 'proxy configs get status: ', res.status, ' url: ', url, ' body: ', res.body)
+
+  if res and res.status == 200 and res.body then
+    local ok, res = pcall(cjson.decode, res.body)
+    if not ok then return nil, res end
+    local json = res
+
+    return json.proxy_configs or array()
+  else
+    ngx.log(ngx.DEBUG, 'proxy configs get error: ', status_code_error(res), ' url: ', url)
+    return nil, 'invalid status'
+  end
+end
+
+function _M:index(host)
+  local http_client = self.http_client
+
+  if not http_client then
+    return nil, 'not initialized'
+  end
+
+  local env = resty_env.value('THREESCALE_DEPLOYMENT_ENV')
+
+  if not env then
+    return nil, 'missing environment'
+  end
+
+  local PROXY_CONFIGS_PER_PAGE = 500
+  -- Keep asking until the results length is different than "per_page" param
+  -- If the 3scale API endpoint version does not support paginations AND
+  -- the number of  results equals to PROXY_CONFIGS_PER_PAGE, the gateway will keep fetching
+  -- configs indefinitely. The 3scale API endpoint version must support pagination to
+  -- avoid endless loop.
+
+  local all_results_per_page = false
+  local current_page = 1
+  local proxy_configs = array()
+
+  repeat
+    local page_proxy_configs, err = proxy_configs_per_page(http_client, self.endpoint, host, env, current_page, PROXY_CONFIGS_PER_PAGE)
+    if not page_proxy_configs and err then
+      return nil, err
+    end
+
+    for _, proxy_config in ipairs(page_proxy_configs) do
+      insert(proxy_configs, proxy_config)
+    end
+
+    all_results_per_page = #page_proxy_configs == PROXY_CONFIGS_PER_PAGE
+    current_page = current_page + 1
+  until(not  all_results_per_page)
+
+  return parse_proxy_configs(self, proxy_configs)
+end
+
 function _M:call(host)
   if self == _M  or not self then
     local m = _M.new()
@@ -271,31 +332,19 @@ function _M:call(host)
   -- When specific version for a specific service is defined,
   -- loading services one by one is required
   --
-  -- APICAST_SERVICE_%s_CONFIGURATION_VERSION does not work then THREESCALE_PORTAL_ENDPOINT
+  -- APICAST_SERVICE_%s_CONFIGURATION_VERSION does not work when the THREESCALE_PORTAL_ENDPOINT
   -- points to master (the API does not allow it), hence error is returned
 
   local use_service_version = is_service_version_set()
-  local use_service_list = is_service_list_set()
-  local use_service_filter_by_url = is_service_filter_by_url_set()
 
   if use_service_version and proxy_config_path then
     return nil, 'APICAST_SERVICE_%s_CONFIGURATION_VERSION cannot be used when proxy config path is provided'
   end
 
-  if use_service_list and proxy_config_path then
-    return nil, 'APICAST_SERVICES_LIST cannot be used when proxy config path is provided'
-  end
-
-  if use_service_filter_by_url and proxy_config_path then
-    return nil, 'APICAST_SERVICES_FILTER_BY_URL cannot be used when proxy config path is provided'
-  end
-
   if use_service_version then
     return self:index_per_service()
-  elseif use_service_list then
-    return self:index_per_service()
-  elseif use_service_filter_by_url then
-    return self:index_per_service()
+  elseif proxy_config_path then
+    return self:index_custom_path(host)
   else
     return self:index(host)
   end
@@ -313,6 +362,38 @@ local services_subset = function()
   end
 end
 
+
+-- Returns existing services in a single page
+-- @param http_client the http client object
+-- @param portal_endpoint 3scale API endpoint
+-- @param page page in the paginated list. Defaults to 1 for the API, as the client will not send the page param.
+-- @param per_page number of results per page. Default and max is 500 for the API, as the client will not send the per_page param.
+local services_per_page = function(http_client, portal_endpoint, page, per_page)
+  local encoded_args = ngx.encode_args({page = page, per_page = per_page})
+  local query_args = encoded_args ~= '' and '?'..encoded_args
+  local base_url = services_index_endpoint(portal_endpoint)
+  local url = query_args and base_url..query_args or base_url
+
+  local res, err = http_client.get(url)
+
+  if not res and err then
+    ngx.log(ngx.DEBUG, 'services get error: ', err, ' url: ', url)
+    return nil, err
+  end
+
+  ngx.log(ngx.DEBUG, 'services get status: ', res.status, ' url: ', url, ' body: ', res.body)
+
+  if res.status == 200 then
+    local ok, res = pcall(cjson.decode, res.body)
+    if not ok then return nil, res end
+    local json = res
+
+    return json.services or array()
+  else
+    return nil, status_code_error(res)
+  end
+end
+
 -- Returns a table with services.
 -- There are 2 cases:
 -- A) with APICAST_SERVICES_LIST. The method returns a table where each element
@@ -341,23 +422,32 @@ function _M:services()
     return nil, 'no endpoint'
   end
 
-  local url = services_index_endpoint(endpoint)
-  local res, err = http_client.get(url)
-
-  if not res and err then
-    ngx.log(ngx.DEBUG, 'services get error: ', err, ' url: ', url)
-    return nil, err
-  end
+  local SERVICES_PER_PAGE = 500
+  -- Keep asking until the results length is different than "per_page" param
+  -- If the 3scale API endpoint version does not support paginations AND
+  -- the number of  results equals to SERVICES_PER_PAGE, the gateway will keep fetching
+  -- services indefinitely. The 3scale API endpoint version must support pagination to
+  -- avoid endless loop.
+
+  local all_results_per_page = false
+  local current_page = 1
+  local services = array()
+
+  repeat
+    local page_services, err = services_per_page(http_client, endpoint, current_page, SERVICES_PER_PAGE)
+    if not page_services and err then
+      return nil, err
+    end
 
-  ngx.log(ngx.DEBUG, 'services get status: ', res.status, ' url: ', url)
+    for _, service in ipairs(page_services) do
+      insert(services, service)
+    end
 
-  if res.status == 200 then
-    local json = cjson.decode(res.body)
+    all_results_per_page = #page_services == SERVICES_PER_PAGE
+    current_page = current_page + 1
+  until(not  all_results_per_page)
 
-    return json.services or array()
-  else
-    return nil, status_code_error(res)
-  end
+  return services
 end
 
 function _M:oidc_issuer_configuration(service)

gateway/src/resty/http_ng/backend/test.lua

@@ -8,9 +8,17 @@ local remove = table.remove
 local error = error
 local format = string.format
 local response = require 'resty.http_ng.response'
+local url_helper = require('resty.url_helper')
 
 local _M = {}
 
+local function parse_url(url)
+    local url_obj = url_helper.parse_url(url)
+    if not url_obj then return end
+    if url_obj.query then url_obj.query = ngx.decode_args(url_obj.query) end
+    return url_obj
+end
+
 local function contains(expected, actual)
   if actual == expected then return true end
   local t1,t2 = type(actual), type(expected)
@@ -25,7 +33,10 @@ local function contains(expected, actual)
 
   if t1 == 'table' then
     for k,v in pairs(expected) do
-      local ok, err = contains(v, actual[k])
+      -- compare urls with query params no matter param order
+      local expected_val = k == 'url' and parse_url(v) or v
+      local actual_val = k == 'url' and parse_url(actual[k]) or actual[k]
+      local ok, err = contains(expected_val, actual_val)
       if not ok then
         return false, format('[%q] %s', k, err)
       end