proxy configs pagination: e2e tests

gateway/src/apicast/configuration_loader/remote_v2.lua

@@ -265,7 +265,7 @@ local proxy_configs_per_page = function(http_client, portal_endpoint, host, env,
     return nil, err
   end
 
-  ngx.log(ngx.DEBUG, 'proxy configs get status: ', res.status, ' url: ', url)
+  ngx.log(ngx.DEBUG, 'proxy configs get status: ', res.status, ' url: ', url, ' body: ', res.body)
 
   if res and res.status == 200 and res.body then
     local ok, res = pcall(cjson.decode, res.body)
@@ -381,7 +381,7 @@ local services_per_page = function(http_client, portal_endpoint, page, per_page)
     return nil, err
   end
 
-  ngx.log(ngx.DEBUG, 'services get status: ', res.status, ' url: ', url)
+  ngx.log(ngx.DEBUG, 'services get status: ', res.status, ' url: ', url, ' body: ', res.body)
 
   if res.status == 200 then
     local ok, res = pcall(cjson.decode, res.body)

t/configuration-loading-boot-remote.t

@@ -42,34 +42,7 @@ GET /t
 {"services":[],"oidc":[]}
 --- exit_code: 200
 
-=== TEST 2: lazy load configuration from remote account/proxy_configs
-endpoint should load that configuration and not fail
---- main_config
-env THREESCALE_PORTAL_ENDPOINT=http://127.0.0.1:$TEST_NGINX_SERVER_PORT;
-env APICAST_CONFIGURATION_LOADER=lazy;
-env THREESCALE_DEPLOYMENT_ENV=foobar;
-env PATH;
---- http_config
-  lua_package_path "$TEST_NGINX_LUA_PATH";
---- config
-location = /t {
-  content_by_lua_block {
-    local loader = require('apicast.configuration_loader.remote_v2')
-    local body = assert(loader:call("example.com"))
-    ngx.say(body)
-  }
-}
-
-location = /admin/api/account/proxy_configs/foobar.json {
-    echo '{}';
-}
---- request
-GET /t
---- expected_json
-{"services":[],"oidc":[]}
---- exit_code: 200
-
-=== TEST 3: retrieve config with liquid values
+=== TEST 2: retrieve config with liquid values
 should not fail
 --- main_config
 env THREESCALE_PORTAL_ENDPOINT=http://127.0.0.1:$TEST_NGINX_SERVER_PORT;

t/configuration-loading-filter-services-by-url.t

@@ -56,17 +56,17 @@ __DATA__
 --- error_code eval
 [200, 404]
 
-=== TEST 2: multi service configuration limited with Regexp Filter with paginated service list
-This test is configured to provide 3 pages of services. On each page, there is one service "one-*"
+=== TEST 2: Regexp Filter with paginated proxy config list
+This test is configured to provide 3 pages of proxy configs. On each page, there is one service "one-*"
 which is valid according to the filter by url. The test will do one request to each valid service.
 --- env eval
 (
-  'APICAST_SERVICES_FILTER_BY_URL' => "^one*",
   'APICAST_CONFIGURATION_LOADER' => 'lazy',
+  'APICAST_SERVICES_FILTER_BY_URL' => "^one*",
   'THREESCALE_PORTAL_ENDPOINT' => "http://test:$ENV{TEST_NGINX_SERVER_PORT}"
 )
 --- upstream env
-location = /admin/api/services.json {
+location /admin/api/account/proxy_configs/production.json {
   content_by_lua_block {
     local args = ngx.req.get_uri_args(0)
     local page = 1
@@ -78,48 +78,34 @@ location = /admin/api/services.json {
       per_page = tonumber(args.per_page)
     end
 
-    local services_per_page = {}
-    for i = (page - 1)*per_page + 1,math.min(page*per_page, 1256)
-    do
-      table.insert(services_per_page, {service = { id = i }})
+    -- this test is designed for pages of 500 items
+    require('luassert').equals(500, per_page)
+    require('luassert').is_true(1 <= page and page < 4)
+
+    local function build_proxy_config(service_id, host)
+      return { proxy_config = {
+        content = { id = service_id, backend_version = 1,
+          proxy = {
+            hosts = { host },
+            api_backend = 'http://test:$TEST_NGINX_SERVER_PORT/api/',
+            backend = { endpoint = 'http://test:$TEST_NGINX_SERVER_PORT' },
+            proxy_rules = { { pattern = '/', http_method = 'GET', metric_system_name = 'test', delta = 1 } }
+          }
+        }
+      }}
     end
 
-    require('luassert').True(#services_per_page <= per_page)
+    local configs_per_page = {}
 
-    local response = { services = services_per_page }
-    ngx.header.content_type = 'application/json;charset=utf-8'
-    ngx.say(require('cjson').encode(response))
-  }
-}
-
-location ~ /admin/api/services/\d+/proxy/configs/production/latest.json {
-  content_by_lua_block {
-    local proxy_config = {
-      content = { id = 1, backend_version = 1,
-        proxy = {
-          hosts = { 'two' },
-          api_backend = 'http://test:$TEST_NGINX_SERVER_PORT/api/',
-          backend = { endpoint = 'http://test:$TEST_NGINX_SERVER_PORT' },
-          proxy_rules = { { pattern = '/', http_method = 'GET', metric_system_name = 'test', delta = 1 } }
-        }
-      }
-    }
+    local host_map = { [1] = 'one-1', [501] = 'one-501', [1001] = 'one-1001' }
 
-    if( ngx.var.uri == '/admin/api/services/1/proxy/configs/production/latest.json' )
-    then
-      proxy_config.content.id = 1
-      proxy_config.content.proxy.hosts = { 'one-1' }
-    elseif( ngx.var.uri == '/admin/api/services/500/proxy/configs/production/latest.json'  )
-    then
-      proxy_config.content.id = 500
-      proxy_config.content.proxy.hosts = { 'one-500' }
-    elseif( ngx.var.uri == '/admin/api/services/1000/proxy/configs/production/latest.json'  )
-    then
-      proxy_config.content.id = 1000
-      proxy_config.content.proxy.hosts = { 'one-1000' }
+    for i = (page - 1)*per_page + 1,math.min(page*per_page, 1256)
+    do
+      local host = host_map[i] or 'two'
+      table.insert(configs_per_page, build_proxy_config(i, host))
     end
 
-    local response = { proxy_config = proxy_config }
+    local response = { proxy_configs = configs_per_page }
     ngx.header.content_type = 'application/json;charset=utf-8'
     ngx.say(require('cjson').encode(response))
   }
@@ -137,7 +123,7 @@ location /transactions/authrep.xml {
 --- pipelined_requests eval
 ["GET /?user_key=1","GET /?user_key=1","GET /?user_key=1","GET /?user_key=2"]
 --- more_headers eval
-["Host: one-1","Host: one-500","Host: one-1000","Host: two"]
+["Host: one-1","Host: one-501","Host: one-1001","Host: two"]
 --- response_body eval
 ["yay, api backend\n","yay, api backend\n","yay, api backend\n",""]
 --- error_code eval

t/configuration-loading-from-service-list.t

@@ -230,81 +230,56 @@ my $jwt = encode_jwt(payload => {
 --- no_error_log
 [error]
 
-=== TEST 4: load a config where only some of the services have an OIDC configuration
-This is a regression test. APIcast crashed when loading a config where only
-some of the services used OIDC.
-The reason is that we created an array of OIDC configs with
-size=number_of_services. Let's say we have 100 services and only the 50th has an
-OIDC config. In this case, we created this Lua table:
-{ [50] = oidc_config_here }.
-The problem is that cjson raises an error when trying to convert a sparse array
-like that into JSON. Using the default cjson configuration, the minimum number
-of elements to reproduce the error is 11. So in this test, we create 11 services
-and assign an OIDC config only to the last one. Check
-https://www.kyne.com.au/~mark/software/lua-cjson-manual.html#encode_sparse_array
-for more details.
-Now we assign to _false_ the elements of the array that do not have an OIDC
-config, so this test should not crash.
+=== TEST 4: servide list filter with paginated proxy config list
+This test is configured to provide 3 pages of proxy configs. On each page, there is only one service
+which is valid according to the filter by service list. The test will do one request to each valid service.
 --- env eval
 (
   'THREESCALE_DEPLOYMENT_ENV' => 'production',
   'APICAST_CONFIGURATION_LOADER' => 'lazy',
-  'APICAST_SERVICES_LIST' => '1,2,3,4,5,6,7,8,9,10,11',
+  'APICAST_SERVICES_LIST' => '1,501,1001',
   'THREESCALE_PORTAL_ENDPOINT' => "http://test:$ENV{TEST_NGINX_SERVER_PORT}"
 )
 --- upstream env
-location ~ /admin/api/services/([0-9]|10)/proxy/configs/production/latest.json {
-echo '
-{
-  "proxy_config": {
-    "content": {
-      "id": 1,
-      "backend_version": 1,
-      "proxy": {
-        "api_backend": "http://test:$TEST_NGINX_SERVER_PORT/api/",
-        "backend": {
-          "endpoint": "http://test:$TEST_NGINX_SERVER_PORT"
-        },
-        "proxy_rules": [
-          {
-            "pattern": "/",
-            "http_method": "GET",
-            "metric_system_name": "test",
-            "delta": 1
+location /admin/api/account/proxy_configs/production.json {
+  content_by_lua_block {
+    local args = ngx.req.get_uri_args(0)
+    local page = 1
+    if args.page then
+      page = tonumber(args.page)
+    end
+    local per_page = 500
+    if args.per_page then
+      per_page = tonumber(args.per_page)
+    end
+
+    -- this test is designed for pages of 500 items
+    require('luassert').equals(500, per_page)
+    require('luassert').is_true(1 <= page and page < 4)
+
+    local function build_proxy_config(service_id, host)
+      return { proxy_config = {
+        content = { id = service_id, backend_version = 1,
+          proxy = {
+            hosts = { host },
+            api_backend = 'http://test:$TEST_NGINX_SERVER_PORT/api/',
+            backend = { endpoint = 'http://test:$TEST_NGINX_SERVER_PORT' },
+            proxy_rules = { { pattern = '/', http_method = 'GET', metric_system_name = 'test', delta = 1 } }
           }
-        ]
-      }
-    }
-  }
-}
-';
-}
+        }
+      }}
+    end
 
-location = /admin/api/services/11/proxy/configs/production/latest.json {
-echo '{ "proxy_config": { "content": { "proxy": { "oidc_issuer_endpoint": "http://test:$TEST_NGINX_SERVER_PORT/issuer/endpoint" } } } }';
-}
+    local configs_per_page = {}
 
-location = /issuer/endpoint/.well-known/openid-configuration {
-  content_by_lua_block {
-    local base = "http://" .. ngx.var.host .. ':' .. ngx.var.server_port
-    ngx.header.content_type = 'application/json;charset=utf-8'
-    ngx.say(require('cjson').encode {
-        issuer = 'https://example.com/auth/realms/apicast',
-        id_token_signing_alg_values_supported = { 'RS256' },
-        jwks_uri = base .. '/jwks',
-    })
-  }
-}
+    for i = (page - 1)*per_page + 1,math.min(page*per_page, 1256)
+    do
+      table.insert(configs_per_page, build_proxy_config(i, 'one-'..tostring(i)))
+    end
 
-location = /jwks {
-  content_by_lua_block {
+    local response = { proxy_configs = configs_per_page }
     ngx.header.content_type = 'application/json;charset=utf-8'
-    ngx.say([[
-        { "keys": [
-            { "kty":"RSA","kid":"somekid",
-              "n":"sKXP3pwND3rkQ1gx9nMb4By7bmWnHYo2kAAsFD5xq0IDn26zv64tjmuNBHpI6BmkLPk8mIo0B1E8MkxdKZeozQ","e":"AQAB" }
-        ] }
-    ]])
+    ngx.say(require('cjson').encode(response))
   }
 }
 
@@ -314,11 +289,14 @@ location /api/ {
 
 --- backend
 location /transactions/authrep.xml {
-content_by_lua_block { ngx.exit(200) }
+  content_by_lua_block { ngx.exit(200) }
 }
 
---- request
-GET /?user_key=uk
---- error_code: 200
---- response_body
-yay, api backend
+--- pipelined_requests eval
+["GET /?user_key=1","GET /?user_key=1","GET /?user_key=1","GET /?user_key=2"]
+--- more_headers eval
+["Host: one-1","Host: one-501","Host: one-1001","Host: one-2"]
+--- response_body eval
+["yay, api backend\n","yay, api backend\n","yay, api backend\n",""]
+--- error_code eval
+[200, 200, 200, 404]

t/configuration-loading-lazy.t

@@ -15,8 +15,8 @@ should just say service is not found
 --- upstream
 location /admin/api/account/proxy_configs/production.json {
   content_by_lua_block {
-    expected = "host=localhost&version=latest"
-    require('luassert').same(ngx.decode_args(expected), ngx.req.get_uri_args(0))
+    local expected = { host = 'localhost', version = 'latest', page = '1', per_page = '500' }
+    require('luassert').same(expected, ngx.req.get_uri_args(0))
 
     ngx.say(require('cjson').encode({}))
   }
@@ -54,8 +54,8 @@ should correctly route the request
 --- upstream env
 location = /admin/api/account/proxy_configs/production.json {
   content_by_lua_block {
-    expected = "host=localhost&version=latest"
-    require('luassert').same(ngx.decode_args(expected), ngx.req.get_uri_args(0))
+    local expected = { host = 'localhost', version = 'latest', page = '1', per_page = '500' }
+    require('luassert').same(expected, ngx.req.get_uri_args(0))
 
     local response = {
       proxy_configs = {
@@ -160,3 +160,72 @@ OIDC url is not valid, uri:
 --- no_error_log
 [error]
 
+=== TEST 6: Load paginated proxy config list
+This test is configured to provide 3 pages of proxy configs.
+The test will try requests with hostnames existing a different pages
+--- env eval
+(
+  'APICAST_CONFIGURATION_LOADER' => 'lazy',
+  'APICAST_SERVICE_CACHE_SIZE' => '2000',
+  'THREESCALE_PORTAL_ENDPOINT' => "http://test:$ENV{TEST_NGINX_SERVER_PORT}"
+)
+--- upstream env
+location /admin/api/account/proxy_configs/production.json {
+  content_by_lua_block {
+    local args = ngx.req.get_uri_args(0)
+    local page = 1
+    if args.page then
+      page = tonumber(args.page)
+    end
+    local per_page = 500
+    if args.per_page then
+      per_page = tonumber(args.per_page)
+    end
+
+    -- this test is designed for pages of 500 items
+    require('luassert').equals(500, per_page)
+    require('luassert').is_true(1 <= page and page < 4)
+
+    local function build_proxy_config(service_id, host)
+      return { proxy_config = {
+        content = { id = service_id, backend_version = 1,
+          proxy = {
+            hosts = { host },
+            api_backend = 'http://test:$TEST_NGINX_SERVER_PORT/api/',
+            backend = { endpoint = 'http://test:$TEST_NGINX_SERVER_PORT' },
+            proxy_rules = { { pattern = '/', http_method = 'GET', metric_system_name = 'test', delta = 1 } }
+          }
+        }
+      }}
+    end
+
+    local configs_per_page = {}
+
+    for i = (page - 1)*per_page + 1,math.min(page*per_page, 1256)
+    do
+      table.insert(configs_per_page, build_proxy_config(i, 'one-'..tostring(i)))
+    end
+
+    local response = { proxy_configs = configs_per_page }
+    ngx.header.content_type = 'application/json;charset=utf-8'
+    ngx.say(require('cjson').encode(response))
+  }
+}
+
+location /api/ {
+  echo 'yay, api backend';
+}
+
+--- backend
+location /transactions/authrep.xml {
+  content_by_lua_block { ngx.exit(200) }
+}
+
+--- pipelined_requests eval
+["GET /?user_key=1","GET /?user_key=1","GET /?user_key=1"]
+--- more_headers eval
+["Host: one-1","Host: one-501","Host: one-1001"]
+--- response_body eval
+["yay, api backend\n","yay, api backend\n","yay, api backend\n"]
+--- error_code eval
+[200, 200, 200]