Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(ssl): support get upstream cert from ssl object #7221

Merged
merged 15 commits into from
Jun 14, 2022
+477 −21
Merged

feat(ssl): support get upstream cert from ssl object #7221

Changes from 14 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b57dc6a
feat(ssl): support get upstream cert frm ssl object
Jun 9, 2022
b3081a4
fix lint
Jun 9, 2022
69735f3
fix lint
Jun 9, 2022
9eeeb83
reindex test case
Jun 9, 2022
6b4ab90
reindex
Jun 9, 2022
249fa7c
change tls_id to tls.clent_cert_id
Jun 9, 2022
8074d8f
remove tls_id
Jun 9, 2022
8b54302
fix doc
Jun 9, 2022
9ad13cc
fix test case
Jun 10, 2022
7386efe
add more test cases
Jun 10, 2022
94de007
remove FIRST
Jun 10, 2022
e2bc606
fix test cases
Jun 10, 2022
f110ee7
fix lint
Jun 10, 2022
98b3ff4
fix doc
Jun 13, 2022
b7570c9
fix doc
Jun 13, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apisix/admin/ssl.lua
View file
Original file line number Diff line number Diff line change
@@ -46,7 +46,7 @@ local function check_conf(id, conf, need_id)
conf.id = id

core.log.info("schema: ", core.json.delay_encode(core.schema.ssl))
core.log.info("conf : ", core.json.delay_encode(conf))
core.log.info("conf: ", core.json.delay_encode(conf))

local ok, err = apisix_ssl.check_ssl_conf(false, conf)
if not ok then
23 changes: 23 additions & 0 deletions apisix/init.lua
View file
Original file line number Diff line number Diff line change
@@ -495,6 +495,29 @@ function _M.http_access_phase()
or route_val.upstream
end

if api_ctx.matched_upstream and api_ctx.matched_upstream.tls and
api_ctx.matched_upstream.tls.client_cert_id then

local cert_id = api_ctx.matched_upstream.tls.client_cert_id
local upstream_ssl = router.router_ssl.get_by_id(cert_id)
if not upstream_ssl or upstream_ssl.type ~= "client" then
local err = upstream_ssl and
"ssl type should be 'client'" or
"ssl id [" .. cert_id .. "] not exits"
core.log.error("failed to get ssl cert: ", err)

if is_http then
return core.response.exit(502)
end

return ngx_exit(1)
end

core.log.info("matched ssl: ",
core.json.delay_encode(upstream_ssl, true))
api_ctx.upstream_ssl = upstream_ssl
end

if enable_websocket then
api_ctx.var.upstream_upgrade = api_ctx.var.http_upgrade
api_ctx.var.upstream_connection = api_ctx.var.http_connection
40 changes: 34 additions & 6 deletions apisix/schema_def.lua
View file
Original file line number Diff line number Diff line change
@@ -404,6 +404,7 @@ local upstream_schema = {
tls = {
type = "object",
properties = {
client_cert_id = id_schema,
client_cert = certificate_scheme,
client_key = private_key_schema,
verify = {
@@ -414,8 +415,17 @@ local upstream_schema = {
},
},
dependencies = {
client_cert = {"client_key"},
client_key = {"client_cert"},
client_cert = {
required = {"client_key"},
["not"] = {required = {"client_cert_id"}}
},
client_key = {
required = {"client_cert"},
["not"] = {required = {"client_cert_id"}}
},
client_cert_id = {
["not"] = {required = {"client_client", "client_key"}}
}
}
},
keepalive_pool = {
@@ -504,7 +514,7 @@ local upstream_schema = {
oneOf = {
{required = {"type", "nodes"}},
{required = {"type", "service_name", "discovery_type"}},
},
}
}

-- TODO: add more nginx variable support
@@ -722,6 +732,14 @@ _M.ssl = {
type = "object",
properties = {
id = id_schema,
type = {
description = "ssl certificate type, " ..
"server to server certificate, " ..
"client to client certificate for upstream",
type = "string",
default = "server",
enum = {"server", "client"}
},
cert = certificate_scheme,
key = private_key_schema,
sni = {
@@ -772,10 +790,20 @@ _M.ssl = {
create_time = timestamp_def,
update_time = timestamp_def
},
oneOf = {
{required = {"sni", "key", "cert"}},
{required = {"snis", "key", "cert"}}
["if"] = {
properties = {
type = {
enum = {"server"},
},
},
},
["then"] = {
oneOf = {
{required = {"sni", "key", "cert"}},
{required = {"snis", "key", "cert"}}
}
},
["else"] = {required = {"key", "cert"}}
}


4 changes: 4 additions & 0 deletions apisix/ssl.lua
View file
Original file line number Diff line number Diff line change
@@ -197,6 +197,10 @@ function _M.check_ssl_conf(in_dp, conf)
return nil, err
end

if conf.type == "client" then
return true
end

local numcerts = conf.certs and #conf.certs or 0
local numkeys = conf.keys and #conf.keys or 0
if numcerts ~= numkeys then
18 changes: 17 additions & 1 deletion apisix/ssl/router/radixtree_sni.lua
View file
Original file line number Diff line number Diff line change
@@ -26,6 +26,7 @@ local error = error
local str_find = core.string.find
local str_gsub = string.gsub
local str_lower = string.lower
local tostring = tostring
local ssl_certificates
local radixtree_router
local radixtree_router_ver
@@ -44,7 +45,7 @@ local function create_router(ssl_items)
local idx = 0

for _, ssl in config_util.iterate_values(ssl_items) do
if ssl.value ~= nil and
if ssl.value ~= nil and ssl.value.type == "server" and
(ssl.value.status == nil or ssl.value.status == 1) then -- compatible with old version

local j = 0
@@ -261,4 +262,19 @@ function _M.init_worker()
end


function _M.get_by_id(ssl_id)
local ssl
local ssls = core.config.fetch_created_obj("/ssl")
if ssls then
ssl = ssls:get(tostring(ssl_id))
end

if not ssl then
return nil
end

return ssl.value
end


return _M
37 changes: 35 additions & 2 deletions apisix/upstream.lua
View file
Original file line number Diff line number Diff line change
@@ -330,14 +330,24 @@ function _M.set_by_route(route, api_ctx)

local scheme = up_conf.scheme
if (scheme == "https" or scheme == "grpcs") and up_conf.tls then

local client_cert, client_key
if up_conf.tls.client_cert_id then
client_cert = api_ctx.upstream_ssl.cert
client_key = api_ctx.upstream_ssl.key
else
client_cert = up_conf.tls.client_cert
client_key = up_conf.tls.client_key
end

-- the sni here is just for logging
local sni = api_ctx.var.upstream_host
local cert, err = apisix_ssl.fetch_cert(sni, up_conf.tls.client_cert)
local cert, err = apisix_ssl.fetch_cert(sni, client_cert)
if not ok then
return 503, err
end

local key, err = apisix_ssl.fetch_pkey(sni, up_conf.tls.client_key)
local key, err = apisix_ssl.fetch_pkey(sni, client_key)
if not ok then
return 503, err
end
@@ -415,6 +425,29 @@ local function check_upstream_conf(in_dp, conf)
return false, "invalid configuration: " .. err
end

local ssl_id = conf.tls and conf.tls.client_cert_id
if ssl_id then
local key = "/ssl/" .. ssl_id
local res, err = core.etcd.get(key)
if not res then
return nil, "failed to fetch ssl info by "
.. "ssl id [" .. ssl_id .. "]: " .. err
end

if res.status ~= 200 then
return nil, "failed to fetch ssl info by "
.. "ssl id [" .. ssl_id .. "], "
.. "response code: " .. res.status
end
if res.body and res.body.node and
res.body.node.value and res.body.node.value.type ~= "client" then

return nil, "failed to fetch ssl info by "
.. "ssl id [" .. ssl_id .. "], "
.. "wrong ssl type"
end
end

-- encrypt the key in the admin
if conf.tls and conf.tls.client_key then
conf.tls.client_key = apisix_ssl.aes_encrypt_pkey(conf.tls.client_key)
8 changes: 6 additions & 2 deletions docs/en/latest/admin-api.md
View file
Original file line number Diff line number Diff line change
@@ -541,8 +541,9 @@ In addition to the equalization algorithm selections, Upstream also supports pas
| labels | optional | Attributes of the Upstream specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} |
| create_time | optional | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 |
| update_time | optional | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 |
| tls.client_cert | optional | Sets the client certificate while connecting to a TLS Upstream. | |
| tls.client_key | optional | Sets the client private key while connecting to a TLS Upstream. | |
| tls.client_cert | optional, can't be used with `tls.client_cert_id` | Sets the client certificate while connecting to a TLS Upstream. | |
| tls.client_key | optional, can't be used with `tls.client_cert_id` | Sets the client private key while connecting to a TLS Upstream. | |
| tls.client_cert_id | optional, can't be used with `tls.client_cert` and `tls.client_key` | Set the referenced [SSL](#ssl) id. | |
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tokers pls confirm this first

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No problem but currently it looks strange as we use an SSL object while the name is client_cert_id.

| keepalive_pool.size | optional | Sets `keepalive` directive dynamically. | |
| keepalive_pool.idle_timeout | optional | Sets `keepalive_timeout` directive dynamically. | |
| keepalive_pool.requests | optional | Sets `keepalive_requests` directive dynamically. | |
@@ -570,6 +571,8 @@ You can set the `scheme` to `tls`, which means "TLS over TCP".

To use mTLS to communicate with Upstream, you can use the `tls.client_cert/key` in the same format as SSL's `cert` and `key` fields.

Or you can reference SSL object by `tls.client_cert_id` to set SSL cert and key. The SSL object can be referenced only if the `type` field is `client`, otherwise the request will be rejected by APISIX. In addition, only `cert` and `key` will be used in the SSL object.

To allow Upstream to have a separate connection pool, use `keepalive_pool`. It can be configured by modifying its child fields.

Example Configuration:
@@ -789,6 +792,7 @@ Currently, the response is returned from etcd.
| labels | False | Match Rules | Attributes of the resource specified as key-value pairs. | {"version":"v2","build":"16","env":"production"} |
| create_time | False | Auxiliary | Epoch timestamp (in seconds) of the created time. If missing, this field will be populated automatically. | 1602883670 |
| update_time | False | Auxiliary | Epoch timestamp (in seconds) of the updated time. If missing, this field will be populated automatically. | 1602883670 |
| type | False | Certificate position | Identifies the type of certificate, default `server`. | `client` Indicates that the certificate is a client certificate, which is used when APISIX accesses the upstream; `server` Indicates that the certificate is a server-side certificate, which is used by APISIX when verifying client requests. |
| status | False | Auxiliary | Enables the current SSL. Set to `1` (enabled) by default. | `1` to enable, `0` to disable |

Example Configuration:
8 changes: 6 additions & 2 deletions docs/zh/latest/admin-api.md
View file
Original file line number Diff line number Diff line change
@@ -549,8 +549,9 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上
| labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} |
| create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 |
| update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 |
| tls.client_cert | 可选 | https 证书 | 设置跟上游通信时的客户端证书，细节见下文 | |
| tls.client_key | 可选 | https 证书私钥 | 设置跟上游通信时的客户端私钥，细节见下文 | |
| tls.client_cert | 可选，不能和 `tls.client_cert_id` 一起使用 | https 证书 | 设置跟上游通信时的客户端证书，细节见下文 | |
| tls.client_key | 可选，不能和 `tls.client_cert_id` 一起使用 | https 证书私钥 | 设置跟上游通信时的客户端私钥，细节见下文 | |
| tls.client_cert_id | 可选，不能和 `tls.client_cert`、`tls.client_key` 一起使用 | SSL | 设置引用的 ssl id，详见 [SSL](#ssl) | |
|keepalive_pool.size | 可选 | 辅助 | 动态设置 `keepalive` 指令，细节见下文 |
|keepalive_pool.idle_timeout | 可选 | 辅助 | 动态设置 `keepalive_timeout` 指令，细节见下文 |
|keepalive_pool.requests | 可选 | 辅助 | 动态设置 `keepalive_requests` 指令，细节见下文 |
@@ -578,6 +579,8 @@ APISIX 的 Upstream 除了基本的负载均衡算法选择外，还支持对上
`tls.client_cert/key` 可以用来跟上游进行 mTLS 通信。
他们的格式和 SSL 对象的 `cert` 和 `key` 一样。

`tls.client_cert_id` 可以用来指定引用的 SSL 对象。只有当 SSL 对象的 `type` 字段为 client 时才能被引用，否则请求会被 APISIX 拒绝。另外，SSL 对象中只有 `cert`和`key` 会被使用。

`keepalive_pool` 允许 upstream 对象有自己单独的连接池。
它下属的字段，比如 `requests`，可以用了配置上游连接保持的参数。
这个特性需要 APISIX 运行于 [APISIX-Base](./FAQ.md#如何构建-APISIX-Base-环境？)。
@@ -799,6 +802,7 @@ $ curl http://127.0.0.1:9080/get
| labels | 可选 | 匹配规则 | 标识附加属性的键值对 | {"version":"v2","build":"16","env":"production"} |
| create_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 |
| update_time | 可选 | 辅助 | 单位为秒的 epoch 时间戳，如果不指定则自动创建 | 1602883670 |
| type | 可选 | 辅助 | 标识证书的类型，缺省为 `server`。 | `client` 表示证书是客户端证书，APISIX 访问上游时使用；`server` 表示证书是服务端证书，APISIX 验证客户端请求时使用 |
| status | 可选 | 辅助 | 是否启用此 SSL，缺省 `1`。 | `1` 表示启用，`0` 表示禁用 |

ssl 对象 json 配置内容：
73 changes: 71 additions & 2 deletions t/admin/ssl.t
View file
Original file line number Diff line number Diff line change
@@ -236,7 +236,7 @@ GET /t
GET /t
--- error_code: 400
--- response_body
{"error_msg":"invalid configuration: value should match only one schema, but matches none"}
{"error_msg":"invalid configuration: then clause did not match"}
--- no_error_log
[error]

@@ -535,7 +535,7 @@ passed
GET /t
--- error_code: 400
--- response_body
{"error_msg":"invalid configuration: value should match only one schema, but matches none"}
{"error_msg":"invalid configuration: then clause did not match"}
--- no_error_log
[error]

@@ -771,3 +771,72 @@ GET /t
passed
--- no_error_log
[error]



=== TEST 20: missing sni information
--- config
location /t {
content_by_lua_block {
local core = require("apisix.core")
local t = require("lib.test_admin")

local ssl_cert = t.read_file("t/certs/apisix.crt")
local ssl_key = t.read_file("t/certs/apisix.key")
local data = {cert = ssl_cert, key = ssl_key}

local code, body = t.test('/apisix/admin/ssl/1',
ngx.HTTP_PUT,
core.json.encode(data),
[[{
"node": {
"key": "/apisix/ssl/1"
},
"action": "set"
}]]
)

ngx.status = code
ngx.print(body)
}
}
--- request
GET /t
--- error_code: 400
--- response_body
{"error_msg":"invalid configuration: then clause did not match"}
--- no_error_log
[error]



=== TEST 21: type client, missing sni information
--- config
location /t {
content_by_lua_block {
local core = require("apisix.core")
local t = require("lib.test_admin")

local ssl_cert = t.read_file("t/certs/apisix.crt")
local ssl_key = t.read_file("t/certs/apisix.key")
local data = {type = "client", cert = ssl_cert, key = ssl_key}

local code, body = t.test('/apisix/admin/ssl/1',
ngx.HTTP_PUT,
core.json.encode(data),
[[{
"node": {
"key": "/apisix/ssl/1"
},
"action": "set"
}]]
)

ngx.status = code
ngx.print(body)
}
}
--- request
GET /t
--- response_body chomp
passed
8 changes: 4 additions & 4 deletions t/admin/ssl2.t
View file
Original file line number Diff line number Diff line change
@@ -71,7 +71,7 @@ __DATA__
}
}
--- response_body
{"action":"create","node":{"value":{"cert":"","key":"","sni":"not-unwanted-post.com","status":1}}}
{"action":"create","node":{"value":{"cert":"","key":"","sni":"not-unwanted-post.com","status":1,"type":"server"}}}
@@ -104,7 +104,7 @@ __DATA__
}
}
--- response_body
{"action":"set","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"test.com","status":1}}}
{"action":"set","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"test.com","status":1,"type":"server"}}}
@@ -137,7 +137,7 @@ __DATA__
}
}
--- response_body
{"action":"compareAndSwap","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"t.com","status":1}}}
{"action":"compareAndSwap","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","key":"","sni":"t.com","status":1,"type":"server"}}}
@@ -172,7 +172,7 @@ __DATA__
}
}
--- response_body
{"action":"get","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","sni":"t.com","status":1}}}
{"action":"get","node":{"key":"/apisix/ssl/1","value":{"cert":"","id":"1","sni":"t.com","status":1,"type":"server"}}}
133 changes: 133 additions & 0 deletions t/admin/upstream.t
View file
Original file line number Diff line number Diff line change
@@ -627,3 +627,136 @@ GET /t
{"error_msg":"wrong upstream id, do not need it"}
--- no_error_log
[error]



=== TEST 19: client_cert/client_key and client_cert_id cannot appear at the same time
--- config
location /t {
content_by_lua_block {
local core = require("apisix.core")
local t = require("lib.test_admin")

local ssl_cert = t.read_file("t/certs/apisix.crt")
local ssl_key = t.read_file("t/certs/apisix.key")
local data = {
nodes = {
["127.0.0.1:8080"] = 1
},
type = "roundrobin",
tls = {
client_cert_id = 1,
client_cert = ssl_cert,
client_key = ssl_key
}
}
local code, body = t.test('/apisix/admin/upstreams',
ngx.HTTP_POST,
core.json.encode(data)
)

ngx.status = code
ngx.print(body)
}
}
--- request
GET /t
--- error_code: 400
--- response_body eval
qr/{"error_msg":"invalid configuration: property \\\"tls\\\" validation failed: failed to validate dependent schema for \\\"client_cert|client_key\\\": value wasn't supposed to match schema"}/
--- no_error_log
[error]



=== TEST 20: tls.client_cert_id does not exist
--- config
location /t {
content_by_lua_block {
local core = require("apisix.core")
local t = require("lib.test_admin")

local data = {
nodes = {
["127.0.0.1:8080"] = 1
},
type = "roundrobin",
tls = {
client_cert_id = 9999999
}
}
local code, body = t.test('/apisix/admin/upstreams',
ngx.HTTP_POST,
core.json.encode(data)
)

ngx.status = code
ngx.print(body)
}
}
--- request
GET /t
--- error_code: 400
--- response_body
{"error_msg":"failed to fetch ssl info by ssl id [9999999], response code: 404"}
--- no_error_log
[error]



=== TEST 21: tls.client_cert_id exist with wrong ssl type
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin")
local json = require("toolkit.json")
local ssl_cert = t.read_file("t/certs/mtls_client.crt")
local ssl_key = t.read_file("t/certs/mtls_client.key")
local data = {
sni = "test.com",
cert = ssl_cert,
key = ssl_key
}
local code, body = t.test('/apisix/admin/ssl/1',
ngx.HTTP_PUT,
json.encode(data)
)

if code >= 300 then
ngx.status = code
ngx.print(body)
return
end

local data = {
upstream = {
scheme = "https",
type = "roundrobin",
nodes = {
["127.0.0.1:1983"] = 1
},
tls = {
client_cert_id = 1
}
},
uri = "/hello"
}
local code, body = t.test('/apisix/admin/routes/1',
ngx.HTTP_PUT,
json.encode(data)
)

if code >= 300 then
ngx.status = code
ngx.print(body)
return
end
}
}
--- request
GET /t
--- error_code: 400
--- response_body
{"error_msg":"failed to fetch ssl info by ssl id [1], wrong ssl type"}
--- no_error_log
[error]
144 changes: 143 additions & 1 deletion t/node/upstream-mtls.t
View file
Original file line number Diff line number Diff line change
@@ -77,7 +77,7 @@ __DATA__
GET /t
--- error_code: 400
--- response_body
{"error_msg":"invalid configuration: property \"upstream\" validation failed: property \"tls\" validation failed: property \"client_key\" is required when \"client_cert\" is set"}
{"error_msg":"invalid configuration: property \"upstream\" validation failed: property \"tls\" validation failed: failed to validate dependent schema for \"client_cert\": property \"client_key\" is required"}
@@ -545,3 +545,145 @@ GET /t
GET /hello_chunked
--- response_body
hello world
=== TEST 13: get cert by tls.client_cert_id
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin")
local json = require("toolkit.json")
local ssl_cert = t.read_file("t/certs/mtls_client.crt")
local ssl_key = t.read_file("t/certs/mtls_client.key")
local data = {
type = "client",
cert = ssl_cert,
key = ssl_key
}
local code, body = t.test('/apisix/admin/ssl/1',
ngx.HTTP_PUT,
json.encode(data)
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
local data = {
upstream = {
scheme = "https",
type = "roundrobin",
nodes = {
["127.0.0.1:1983"] = 1,
},
tls = {
client_cert_id = 1
}
},
uri = "/hello"
}
local code, body = t.test('/apisix/admin/routes/1',
ngx.HTTP_PUT,
json.encode(data)
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
}
}
--- request
GET /t
=== TEST 14: hit
--- upstream_server_config
ssl_client_certificate ../../certs/mtls_ca.crt;
ssl_verify_client on;
--- request
GET /hello
--- response_body
hello world
=== TEST 15: change ssl object type
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin")
local json = require("toolkit.json")
local ssl_cert = t.read_file("t/certs/mtls_client.crt")
local ssl_key = t.read_file("t/certs/mtls_client.key")
local data = {
type = "server",
sni = "test.com",
cert = ssl_cert,
key = ssl_key
}
local code, body = t.test('/apisix/admin/ssl/1',
ngx.HTTP_PUT,
json.encode(data)
)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
}
}
--- request
GET /t
=== TEST 16: hit, ssl object type mismatch
--- upstream_server_config
ssl_client_certificate ../../certs/mtls_ca.crt;
ssl_verify_client on;
--- request
GET /hello
--- error_code: 502
--- error_log
failed to get ssl cert: ssl type should be 'client'
=== TEST 17: delete ssl object
--- config
location /t {
content_by_lua_block {
local t = require("lib.test_admin")
local json = require("toolkit.json")
local code, body = t.test('/apisix/admin/ssl/1', ngx.HTTP_DELETE)
if code >= 300 then
ngx.status = code
ngx.say(body)
return
end
}
}
--- request
GET /t
=== TEST 18: hit, ssl object not exits
--- upstream_server_config
ssl_client_certificate ../../certs/mtls_ca.crt;
ssl_verify_client on;
--- request
GET /hello
--- error_code: 502
--- error_log
failed to get ssl cert: ssl id [1] not exits