From dc8505ee8b98d0f7df3bb88ffa3411fe9742659b Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 30 Nov 2022 18:32:33 +0800 Subject: [PATCH 01/19] feat(prometheus): support collect metrics works in the priviledged agent --- apisix/cli/ngx_tpl.lua | 4 +-- apisix/init.lua | 6 +++++ apisix/plugins/prometheus/exporter.lua | 9 +++++++ docs/en/latest/plugins/prometheus.md | 3 +++ docs/zh/latest/plugins/prometheus.md | 3 +++ t/cli/test_prometheus.sh | 36 ++++++++++++++++++++++++++ t/cli/test_prometheus_stream.sh | 22 ++++++++++++++++ 7 files changed, 81 insertions(+), 2 deletions(-) diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index d63f713dbc46..b4fe044f4f62 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -80,7 +80,7 @@ http { } server { - listen {* prometheus_server_addr *}; + listen {* prometheus_server_addr *} enable_process=privileged_agent; access_log off; @@ -469,7 +469,7 @@ http { {% if enabled_plugins["prometheus"] and prometheus_server_addr then %} server { - listen {* prometheus_server_addr *}; + listen {* prometheus_server_addr *} enable_process=privileged_agent; access_log off; diff --git a/apisix/init.lua b/apisix/init.lua index fbb090bee3f6..d4b912c4b2df 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -870,6 +870,12 @@ function _M.stream_init(args) core.resolver.init_resolver(args) + local process = require("ngx.process") + local ok, err = process.enable_privileged_agent() + if not ok then + core.log.error("failed to enable privileged_agent: ", err) + end + if core.config.init then local ok, err = core.config.init() if not ok then diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua index 45ff94c3f631..5dfb5d285d22 100644 --- a/apisix/plugins/prometheus/exporter.lua +++ b/apisix/plugins/prometheus/exporter.lua @@ -410,7 +410,16 @@ local function shared_dict_status() end +-- for test +local function log_process_type() + local process_type = require("ngx.process").type() + return process_type +end + + local function collect(ctx, stream_only) + core.log.info("process type: ", core.log.delay_exec(log_process_type)) + if not prometheus or not metrics then core.log.error("prometheus: plugin is not initialized, please make sure ", " 'prometheus_metrics' shared dict is present in nginx template") diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md index aaf526b42c2c..56c9e1f4df4f 100644 --- a/docs/en/latest/plugins/prometheus.md +++ b/docs/en/latest/plugins/prometheus.md @@ -102,6 +102,9 @@ plugin_attr: You can then expose it by using the [public-api](public-api.md) Plugin. +Note: If the Prometheus plugin collects too many metrics, it will take CPU resources to calculate the metric data when getting the metrics via URI, which may affect APISIX to process normal requests. To solve this problem, APISIX exposes the URI and calculates the metrics in the priviledged agent. +If the URI is exposed using the public-api plugin, then APISIX will calculate the metric data in a normal worker process, which may still affect APISIX processing of normal requests. + ## Enabling the Plugin The `prometheus` Plugin can be enabled with an empty table. diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md index 5ff15ca499a5..0ce5f756089c 100644 --- a/docs/zh/latest/plugins/prometheus.md +++ b/docs/zh/latest/plugins/prometheus.md @@ -85,6 +85,9 @@ plugin_attr: 你可以使用 [public-api](../../../en/latest/plugins/public-api.md) 插件来暴露该 URI。 +注意：如果 Prometheus 插件收集的指标数量过多，在通过 URI 获取指标时，会占用 CPU 资源来计算指标数据，可能会影响 APISIX 处理正常请求。为解决此问题，APISIX 在 priviledged agent 中暴露 URI 并且计算指标。 +如果使用 public-api 插件暴露该 URI，那么 APISIX 将在普通的 worker 进程中计算指标数据，这仍可能会影响 APISIX 处理正常请求。 + ## 启用插件 `prometheus` 插件可以使用空表 `{}` 开启。 diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index 15f54f9114ee..f3a1b0abb884 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -64,7 +64,11 @@ make stop echo "passed: should listen at configured prometheus address" +rm logs/error.log || true + echo ' +nginx_config: + error_log_level: info plugin_attr: prometheus: enable_export_server: false @@ -108,6 +112,14 @@ make stop echo "passed: should listen at previous prometheus address" +# if get metrics from public API, then collect metrics would run in worker process +if ! grep -E "process type: worker" logs/error.log; then + echo "failed: prometheus should work in worker process when get metrics from public API" + exit 1 +fi + +echo "prometheus should work in worker process when get metrics from public API successfully" + echo ' plugin_attr: prometheus: @@ -175,3 +187,27 @@ fi make stop echo "passed: should use custom metric prefix" + +# collect metrics run in privileged agent +rm logs/error.log || true + +echo ' +nginx_config: + error_log_level: info +' > conf/config.yaml + +make init +make run + +sleep 0.5 + +curl -s -o /dev/null http://127.0.0.1:9091/apisix/prometheus/metrics + +sleep 0.1 + +if ! grep -E "process type: privileged agent" logs/error.log; then + echo "failed: prometheus works well in privileged agent" + exit 1 +fi + +echo "prometheus works well in privileged agent successfully" diff --git a/t/cli/test_prometheus_stream.sh b/t/cli/test_prometheus_stream.sh index 561b9a820cf5..6f699f19e619 100755 --- a/t/cli/test_prometheus_stream.sh +++ b/t/cli/test_prometheus_stream.sh @@ -21,6 +21,8 @@ exit_if_not_customed_nginx +rm logs/error.log || true + echo " apisix: enable_admin: true @@ -29,6 +31,8 @@ apisix: - addr: 9100 stream_plugins: - prometheus +nginx_config: + error_log_level: info " > conf/config.yaml make run @@ -63,6 +67,15 @@ make stop echo "passed: prometheus works when both http & stream are enabled" +if ! grep -E " process type: privileged agent" logs/error.log; then + echo "failed: prometheus run in privileged can't work when both http & stream are enabled" + exit 1 +fi + +echo "passed: prometheus run in privileged works when both http & stream are enabled" + +rm logs/error.log || true + echo " apisix: enable_admin: false @@ -71,6 +84,8 @@ apisix: - addr: 9100 stream_plugins: - prometheus +nginx_config: + error_log_level: info " > conf/config.yaml make run @@ -91,3 +106,10 @@ if ! echo "$out" | grep "apisix_node_info{hostname=" > /dev/null; then fi echo "passed: prometheus works when only stream is enabled" + +if ! grep -E " process type: privileged agent" logs/error.log; then + echo "failed: prometheus run in privileged can't work when only stream is enabled" + exit 1 +fi + +echo "passed: prometheus run in privileged works when only stream is enabled" From bfaf952955a5a2eb4e74f2ccfd4db0d258217e9c Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 30 Nov 2022 18:34:58 +0800 Subject: [PATCH 02/19] fix lint --- docs/en/latest/plugins/prometheus.md | 2 +- docs/zh/latest/plugins/prometheus.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md index 56c9e1f4df4f..41a518ef05e0 100644 --- a/docs/en/latest/plugins/prometheus.md +++ b/docs/en/latest/plugins/prometheus.md @@ -102,7 +102,7 @@ plugin_attr: You can then expose it by using the [public-api](public-api.md) Plugin. -Note: If the Prometheus plugin collects too many metrics, it will take CPU resources to calculate the metric data when getting the metrics via URI, which may affect APISIX to process normal requests. To solve this problem, APISIX exposes the URI and calculates the metrics in the priviledged agent. +Note: If the Prometheus plugin collects too many metrics, it will take CPU resources to calculate the metric data when getting the metrics via URI, which may affect APISIX to process normal requests. To solve this problem, APISIX exposes the URI and calculates the metrics in the privileged agent. If the URI is exposed using the public-api plugin, then APISIX will calculate the metric data in a normal worker process, which may still affect APISIX processing of normal requests. ## Enabling the Plugin diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md index 0ce5f756089c..a74784427c7d 100644 --- a/docs/zh/latest/plugins/prometheus.md +++ b/docs/zh/latest/plugins/prometheus.md @@ -85,7 +85,7 @@ plugin_attr: 你可以使用 [public-api](../../../en/latest/plugins/public-api.md) 插件来暴露该 URI。 -注意：如果 Prometheus 插件收集的指标数量过多，在通过 URI 获取指标时，会占用 CPU 资源来计算指标数据，可能会影响 APISIX 处理正常请求。为解决此问题，APISIX 在 priviledged agent 中暴露 URI 并且计算指标。 +注意：如果 Prometheus 插件收集的指标数量过多，在通过 URI 获取指标时，会占用 CPU 资源来计算指标数据，可能会影响 APISIX 处理正常请求。为解决此问题，APISIX 在 privileged agent 中暴露 URI 并且计算指标。 如果使用 public-api 插件暴露该 URI，那么 APISIX 将在普通的 worker 进程中计算指标数据，这仍可能会影响 APISIX 处理正常请求。 ## 启用插件 From c8534d810a0f2278a48c4351c54d1c5689958950 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 30 Nov 2022 18:39:51 +0800 Subject: [PATCH 03/19] fix lint --- apisix/plugins/prometheus/exporter.lua | 1 + 1 file changed, 1 insertion(+) diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua index 5dfb5d285d22..be466525a0ef 100644 --- a/apisix/plugins/prometheus/exporter.lua +++ b/apisix/plugins/prometheus/exporter.lua @@ -14,6 +14,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- +local require = require local base_prometheus = require("prometheus") local core = require("apisix.core") local plugin = require("apisix.plugin") From aa328fc7b3988c61565ab0fc1d31f772b49f5fb0 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 30 Nov 2022 19:04:09 +0800 Subject: [PATCH 04/19] exit_if_not_customed_nginx --- t/cli/test_prometheus.sh | 36 ------------------ t/cli/test_prometheus_run_in_privileged.sh | 44 ++++++++++++++++++++++ 2 files changed, 44 insertions(+), 36 deletions(-) create mode 100755 t/cli/test_prometheus_run_in_privileged.sh diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index f3a1b0abb884..15f54f9114ee 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -64,11 +64,7 @@ make stop echo "passed: should listen at configured prometheus address" -rm logs/error.log || true - echo ' -nginx_config: - error_log_level: info plugin_attr: prometheus: enable_export_server: false @@ -112,14 +108,6 @@ make stop echo "passed: should listen at previous prometheus address" -# if get metrics from public API, then collect metrics would run in worker process -if ! grep -E "process type: worker" logs/error.log; then - echo "failed: prometheus should work in worker process when get metrics from public API" - exit 1 -fi - -echo "prometheus should work in worker process when get metrics from public API successfully" - echo ' plugin_attr: prometheus: @@ -187,27 +175,3 @@ fi make stop echo "passed: should use custom metric prefix" - -# collect metrics run in privileged agent -rm logs/error.log || true - -echo ' -nginx_config: - error_log_level: info -' > conf/config.yaml - -make init -make run - -sleep 0.5 - -curl -s -o /dev/null http://127.0.0.1:9091/apisix/prometheus/metrics - -sleep 0.1 - -if ! grep -E "process type: privileged agent" logs/error.log; then - echo "failed: prometheus works well in privileged agent" - exit 1 -fi - -echo "prometheus works well in privileged agent successfully" diff --git a/t/cli/test_prometheus_run_in_privileged.sh b/t/cli/test_prometheus_run_in_privileged.sh new file mode 100755 index 000000000000..8b03566862bc --- /dev/null +++ b/t/cli/test_prometheus_run_in_privileged.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +. ./t/cli/common.sh + +git checkout conf/config.yaml + +exit_if_not_customed_nginx + +# collect metrics run in privileged agent +rm logs/error.log || true + +echo ' +nginx_config: + error_log_level: info +' > conf/config.yaml + +make init +make run + +curl -s -o /dev/null http://127.0.0.1:9091/apisix/prometheus/metrics + +if ! grep -E "process type: privileged agent" logs/error.log; then + echo "failed: prometheus works well in privileged agent" + exit 1 +fi + +echo "prometheus works well in privileged agent successfully" From 4e1faf2725015ddf8ae24a247222f4ff13386566 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Wed, 30 Nov 2022 19:45:16 +0800 Subject: [PATCH 05/19] use_apisix_openresty in ngx_tpl --- apisix/cli/ngx_tpl.lua | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index b4fe044f4f62..926984dbd304 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -80,7 +80,11 @@ http { } server { - listen {* prometheus_server_addr *} enable_process=privileged_agent; + {% if use_apisix_openresty then %} + listen {* prometheus_server_addr *} enable_process=privileged_agent; + {% else %} + listen {* prometheus_server_addr *}; + {% end %} access_log off; @@ -469,7 +473,11 @@ http { {% if enabled_plugins["prometheus"] and prometheus_server_addr then %} server { - listen {* prometheus_server_addr *} enable_process=privileged_agent; + {% if use_apisix_openresty then %} + listen {* prometheus_server_addr *} enable_process=privileged_agent; + {% else %} + listen {* prometheus_server_addr *}; + {% end %} access_log off; From 5e0df8b800207ddf9383a0e516f4ea3830372c81 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 1 Dec 2022 10:58:44 +0800 Subject: [PATCH 06/19] move enable privileged agent to ngx_tpl --- apisix/cli/ngx_tpl.lua | 11 +++++++++++ apisix/init.lua | 6 ------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index 926984dbd304..8e4263081c3e 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -75,6 +75,17 @@ http { .. [=[{*lua_cpath*};"; {% if enabled_stream_plugins["prometheus"] then %} + + init_by_lua_block { + require "resty.core" + apisix = require("apisix") + local process = require("ngx.process") + local ok, err = process.enable_privileged_agent() + if not ok then + ngx.log(ngx.ERR, "failed to enable privileged_agent: ", err) + end + } + init_worker_by_lua_block { require("apisix.plugins.prometheus.exporter").http_init(true) } diff --git a/apisix/init.lua b/apisix/init.lua index d4b912c4b2df..fbb090bee3f6 100644 --- a/apisix/init.lua +++ b/apisix/init.lua @@ -870,12 +870,6 @@ function _M.stream_init(args) core.resolver.init_resolver(args) - local process = require("ngx.process") - local ok, err = process.enable_privileged_agent() - if not ok then - core.log.error("failed to enable privileged_agent: ", err) - end - if core.config.init then local ok, err = core.config.init() if not ok then From dc9bdff406f4fbab79a1c48c1585993d89503c18 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 1 Dec 2022 11:13:59 +0800 Subject: [PATCH 07/19] sleep --- t/cli/test_prometheus.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index 15f54f9114ee..2423bfb3f980 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -22,6 +22,7 @@ git checkout conf/config.yaml make run +sleep 0.5 code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/prometheus/metrics) if [ ! $code -eq 404 ]; then From b616feddd8daa2383dd7440c7d6d580491c6c423 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 1 Dec 2022 11:43:38 +0800 Subject: [PATCH 08/19] fix --- t/cli/test_prometheus.sh | 294 +++++++++++++++++++-------------------- 1 file changed, 147 insertions(+), 147 deletions(-) diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index 2423bfb3f980..28371c4cf756 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -21,8 +21,8 @@ git checkout conf/config.yaml +make stop || true make run -sleep 0.5 code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/prometheus/metrics) if [ ! $code -eq 404 ]; then @@ -30,149 +30,149 @@ if [ ! $code -eq 404 ]; then exit 1 fi -code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9091/apisix/prometheus/metrics) -if [ ! $code -eq 200 ]; then - echo "failed: should listen at default prometheus address" - exit 1 -fi - -if ! curl -i http://127.0.0.1:9091/apisix/prometheus/metrics | grep "apisix_nginx_http_current_connections" > /dev/null; then - echo "failed: should listen at default prometheus address" - exit 1 -fi - -make stop - -echo "passed: should listen at default prometheus address" - -echo ' -plugin_attr: - prometheus: - export_addr: - ip: ${{IP}} - port: ${{PORT}} -' > conf/config.yaml - -IP=127.0.0.1 PORT=9092 make run - -code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9092/apisix/prometheus/metrics) -if [ ! $code -eq 200 ]; then - echo "failed: should listen at configured prometheus address" - exit 1 -fi - -make stop - -echo "passed: should listen at configured prometheus address" - -echo ' -plugin_attr: - prometheus: - enable_export_server: false - export_uri: /prometheus/metrics - export_addr: - ip: ${{IP}} - port: ${{PORT}} -' > conf/config.yaml - -IP=127.0.0.1 PORT=9092 make run - -# initialize prometheus metrics public API route #1 -code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} -X PUT http://127.0.0.1:9180/apisix/admin/routes/metrics1 \ - -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \ - -d "{ - \"uri\": \"/prometheus/metrics\", - \"plugins\": { - \"public-api\": {} - } - }") -if [ ! $code -eq 201 ]; then - echo "failed: initialize prometheus metrics public API failed #1" - exit 1 -fi - -sleep 0.5 - -code=$(curl -v -k -i -m 20 -o /dev/null -s http://127.0.0.1:9092/prometheus/metrics || echo 'ouch') -if [ "$code" != "ouch" ]; then - echo "failed: should listen at previous prometheus address" - exit 1 -fi - -code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/prometheus/metrics) -if [ ! $code -eq 200 ]; then - echo "failed: should listen at previous prometheus address" - exit 1 -fi - -make stop - -echo "passed: should listen at previous prometheus address" - -echo ' -plugin_attr: - prometheus: - export_addr: - ip: ${{IP}} - port: ${{PORT}} -' > conf/config.yaml - -out=$(IP=127.0.0.1 PORT=9090 make init 2>&1 || true) -if ! echo "$out" | grep "prometheus port 9090 conflicts with control"; then - echo "failed: can't detect port conflicts" - exit 1 -fi - -echo ' -apisix: - node_listen: ${{PORT}} -plugin_attr: - prometheus: - export_addr: - ip: ${{IP}} - port: ${{PORT}} -' > conf/config.yaml - -out=$(IP=127.0.0.1 PORT=9092 make init 2>&1 || true) -if ! echo "$out" | grep "http listen port 9092 conflicts with prometheus"; then - echo "failed: can't detect port conflicts" - exit 1 -fi - -echo "passed: should detect port conflicts" - -echo ' -plugin_attr: - prometheus: - metric_prefix: apisix_ci_prefix_ - export_addr: - ip: ${{IP}} - port: ${{PORT}} -' > conf/config.yaml - -IP=127.0.0.1 PORT=9092 make run - -# initialize prometheus metrics public API route #2 -code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} -X PUT http://127.0.0.1:9180/apisix/admin/routes/metrics2 \ - -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \ - -d "{ - \"uri\": \"/apisix/prometheus/metrics\", - \"plugins\": { - \"public-api\": {} - } - }") -if [ ! $code -eq 201 ]; then - echo "failed: initialize prometheus metrics public API failed #2" - exit 1 -fi - -sleep 0.5 - -if ! curl -s http://127.0.0.1:9092/apisix/prometheus/metrics | grep "apisix_ci_prefix_" | wc -l; then - echo "failed: should use custom metric prefix" - exit 1 -fi - -make stop - -echo "passed: should use custom metric prefix" +# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9091/apisix/prometheus/metrics) +# if [ ! $code -eq 200 ]; then +# echo "failed: should listen at default prometheus address" +# exit 1 +# fi + +# if ! curl -i http://127.0.0.1:9091/apisix/prometheus/metrics | grep "apisix_nginx_http_current_connections" > /dev/null; then +# echo "failed: should listen at default prometheus address" +# exit 1 +# fi + +# make stop + +# echo "passed: should listen at default prometheus address" + +# echo ' +# plugin_attr: +# prometheus: +# export_addr: +# ip: ${{IP}} +# port: ${{PORT}} +# ' > conf/config.yaml + +# IP=127.0.0.1 PORT=9092 make run + +# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9092/apisix/prometheus/metrics) +# if [ ! $code -eq 200 ]; then +# echo "failed: should listen at configured prometheus address" +# exit 1 +# fi + +# make stop + +# echo "passed: should listen at configured prometheus address" + +# echo ' +# plugin_attr: +# prometheus: +# enable_export_server: false +# export_uri: /prometheus/metrics +# export_addr: +# ip: ${{IP}} +# port: ${{PORT}} +# ' > conf/config.yaml + +# IP=127.0.0.1 PORT=9092 make run + +# # initialize prometheus metrics public API route #1 +# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} -X PUT http://127.0.0.1:9180/apisix/admin/routes/metrics1 \ +# -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \ +# -d "{ +# \"uri\": \"/prometheus/metrics\", +# \"plugins\": { +# \"public-api\": {} +# } +# }") +# if [ ! $code -eq 201 ]; then +# echo "failed: initialize prometheus metrics public API failed #1" +# exit 1 +# fi + +# sleep 0.5 + +# code=$(curl -v -k -i -m 20 -o /dev/null -s http://127.0.0.1:9092/prometheus/metrics || echo 'ouch') +# if [ "$code" != "ouch" ]; then +# echo "failed: should listen at previous prometheus address" +# exit 1 +# fi + +# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/prometheus/metrics) +# if [ ! $code -eq 200 ]; then +# echo "failed: should listen at previous prometheus address" +# exit 1 +# fi + +# make stop + +# echo "passed: should listen at previous prometheus address" + +# echo ' +# plugin_attr: +# prometheus: +# export_addr: +# ip: ${{IP}} +# port: ${{PORT}} +# ' > conf/config.yaml + +# out=$(IP=127.0.0.1 PORT=9090 make init 2>&1 || true) +# if ! echo "$out" | grep "prometheus port 9090 conflicts with control"; then +# echo "failed: can't detect port conflicts" +# exit 1 +# fi + +# echo ' +# apisix: +# node_listen: ${{PORT}} +# plugin_attr: +# prometheus: +# export_addr: +# ip: ${{IP}} +# port: ${{PORT}} +# ' > conf/config.yaml + +# out=$(IP=127.0.0.1 PORT=9092 make init 2>&1 || true) +# if ! echo "$out" | grep "http listen port 9092 conflicts with prometheus"; then +# echo "failed: can't detect port conflicts" +# exit 1 +# fi + +# echo "passed: should detect port conflicts" + +# echo ' +# plugin_attr: +# prometheus: +# metric_prefix: apisix_ci_prefix_ +# export_addr: +# ip: ${{IP}} +# port: ${{PORT}} +# ' > conf/config.yaml + +# IP=127.0.0.1 PORT=9092 make run + +# # initialize prometheus metrics public API route #2 +# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} -X PUT http://127.0.0.1:9180/apisix/admin/routes/metrics2 \ +# -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \ +# -d "{ +# \"uri\": \"/apisix/prometheus/metrics\", +# \"plugins\": { +# \"public-api\": {} +# } +# }") +# if [ ! $code -eq 201 ]; then +# echo "failed: initialize prometheus metrics public API failed #2" +# exit 1 +# fi + +# sleep 0.5 + +# if ! curl -s http://127.0.0.1:9092/apisix/prometheus/metrics | grep "apisix_ci_prefix_" | wc -l; then +# echo "failed: should use custom metric prefix" +# exit 1 +# fi + +# make stop + +# echo "passed: should use custom metric prefix" From fce28d3834102dee50a0d210872c310081194e76 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 1 Dec 2022 17:44:32 +0800 Subject: [PATCH 09/19] debug --- .github/workflows/cli.yml | 21 +-- t/cli/test_prometheus.sh | 293 +++++++++++++++++++------------------- 2 files changed, 158 insertions(+), 156 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 33c170af3457..1fcc20d5b5b4 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -2,15 +2,15 @@ name: CLI Test on: push: - branches: [master, 'release/**'] - paths-ignore: - - 'docs/**' - - '**/*.md' - pull_request: - branches: [master, 'release/**'] - paths-ignore: - - 'docs/**' - - '**/*.md' + # branches: [master, 'release/**'] + # paths-ignore: + # - 'docs/**' + # - '**/*.md' + # pull_request: + # branches: [master, 'release/**'] + # paths-ignore: + # - 'docs/**' + # - '**/*.md' concurrency: group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} @@ -62,5 +62,8 @@ jobs: sudo --preserve-env=OPENRESTY_VERSION \ ./ci/${{ matrix.job_name }}_runner.sh do_install + - name: Setup tmate session + uses: mxschmitt/action-tmate@v3 + - name: Linux Script run: sudo ./ci/${{ matrix.job_name }}_runner.sh script diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index 28371c4cf756..15f54f9114ee 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -21,7 +21,6 @@ git checkout conf/config.yaml -make stop || true make run code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/prometheus/metrics) @@ -30,149 +29,149 @@ if [ ! $code -eq 404 ]; then exit 1 fi -# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9091/apisix/prometheus/metrics) -# if [ ! $code -eq 200 ]; then -# echo "failed: should listen at default prometheus address" -# exit 1 -# fi - -# if ! curl -i http://127.0.0.1:9091/apisix/prometheus/metrics | grep "apisix_nginx_http_current_connections" > /dev/null; then -# echo "failed: should listen at default prometheus address" -# exit 1 -# fi - -# make stop - -# echo "passed: should listen at default prometheus address" - -# echo ' -# plugin_attr: -# prometheus: -# export_addr: -# ip: ${{IP}} -# port: ${{PORT}} -# ' > conf/config.yaml - -# IP=127.0.0.1 PORT=9092 make run - -# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9092/apisix/prometheus/metrics) -# if [ ! $code -eq 200 ]; then -# echo "failed: should listen at configured prometheus address" -# exit 1 -# fi - -# make stop - -# echo "passed: should listen at configured prometheus address" - -# echo ' -# plugin_attr: -# prometheus: -# enable_export_server: false -# export_uri: /prometheus/metrics -# export_addr: -# ip: ${{IP}} -# port: ${{PORT}} -# ' > conf/config.yaml - -# IP=127.0.0.1 PORT=9092 make run - -# # initialize prometheus metrics public API route #1 -# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} -X PUT http://127.0.0.1:9180/apisix/admin/routes/metrics1 \ -# -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \ -# -d "{ -# \"uri\": \"/prometheus/metrics\", -# \"plugins\": { -# \"public-api\": {} -# } -# }") -# if [ ! $code -eq 201 ]; then -# echo "failed: initialize prometheus metrics public API failed #1" -# exit 1 -# fi - -# sleep 0.5 - -# code=$(curl -v -k -i -m 20 -o /dev/null -s http://127.0.0.1:9092/prometheus/metrics || echo 'ouch') -# if [ "$code" != "ouch" ]; then -# echo "failed: should listen at previous prometheus address" -# exit 1 -# fi - -# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/prometheus/metrics) -# if [ ! $code -eq 200 ]; then -# echo "failed: should listen at previous prometheus address" -# exit 1 -# fi - -# make stop - -# echo "passed: should listen at previous prometheus address" - -# echo ' -# plugin_attr: -# prometheus: -# export_addr: -# ip: ${{IP}} -# port: ${{PORT}} -# ' > conf/config.yaml - -# out=$(IP=127.0.0.1 PORT=9090 make init 2>&1 || true) -# if ! echo "$out" | grep "prometheus port 9090 conflicts with control"; then -# echo "failed: can't detect port conflicts" -# exit 1 -# fi - -# echo ' -# apisix: -# node_listen: ${{PORT}} -# plugin_attr: -# prometheus: -# export_addr: -# ip: ${{IP}} -# port: ${{PORT}} -# ' > conf/config.yaml - -# out=$(IP=127.0.0.1 PORT=9092 make init 2>&1 || true) -# if ! echo "$out" | grep "http listen port 9092 conflicts with prometheus"; then -# echo "failed: can't detect port conflicts" -# exit 1 -# fi - -# echo "passed: should detect port conflicts" - -# echo ' -# plugin_attr: -# prometheus: -# metric_prefix: apisix_ci_prefix_ -# export_addr: -# ip: ${{IP}} -# port: ${{PORT}} -# ' > conf/config.yaml - -# IP=127.0.0.1 PORT=9092 make run - -# # initialize prometheus metrics public API route #2 -# code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} -X PUT http://127.0.0.1:9180/apisix/admin/routes/metrics2 \ -# -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \ -# -d "{ -# \"uri\": \"/apisix/prometheus/metrics\", -# \"plugins\": { -# \"public-api\": {} -# } -# }") -# if [ ! $code -eq 201 ]; then -# echo "failed: initialize prometheus metrics public API failed #2" -# exit 1 -# fi - -# sleep 0.5 - -# if ! curl -s http://127.0.0.1:9092/apisix/prometheus/metrics | grep "apisix_ci_prefix_" | wc -l; then -# echo "failed: should use custom metric prefix" -# exit 1 -# fi - -# make stop - -# echo "passed: should use custom metric prefix" +code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9091/apisix/prometheus/metrics) +if [ ! $code -eq 200 ]; then + echo "failed: should listen at default prometheus address" + exit 1 +fi + +if ! curl -i http://127.0.0.1:9091/apisix/prometheus/metrics | grep "apisix_nginx_http_current_connections" > /dev/null; then + echo "failed: should listen at default prometheus address" + exit 1 +fi + +make stop + +echo "passed: should listen at default prometheus address" + +echo ' +plugin_attr: + prometheus: + export_addr: + ip: ${{IP}} + port: ${{PORT}} +' > conf/config.yaml + +IP=127.0.0.1 PORT=9092 make run + +code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9092/apisix/prometheus/metrics) +if [ ! $code -eq 200 ]; then + echo "failed: should listen at configured prometheus address" + exit 1 +fi + +make stop + +echo "passed: should listen at configured prometheus address" + +echo ' +plugin_attr: + prometheus: + enable_export_server: false + export_uri: /prometheus/metrics + export_addr: + ip: ${{IP}} + port: ${{PORT}} +' > conf/config.yaml + +IP=127.0.0.1 PORT=9092 make run + +# initialize prometheus metrics public API route #1 +code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} -X PUT http://127.0.0.1:9180/apisix/admin/routes/metrics1 \ + -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \ + -d "{ + \"uri\": \"/prometheus/metrics\", + \"plugins\": { + \"public-api\": {} + } + }") +if [ ! $code -eq 201 ]; then + echo "failed: initialize prometheus metrics public API failed #1" + exit 1 +fi + +sleep 0.5 + +code=$(curl -v -k -i -m 20 -o /dev/null -s http://127.0.0.1:9092/prometheus/metrics || echo 'ouch') +if [ "$code" != "ouch" ]; then + echo "failed: should listen at previous prometheus address" + exit 1 +fi + +code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/prometheus/metrics) +if [ ! $code -eq 200 ]; then + echo "failed: should listen at previous prometheus address" + exit 1 +fi + +make stop + +echo "passed: should listen at previous prometheus address" + +echo ' +plugin_attr: + prometheus: + export_addr: + ip: ${{IP}} + port: ${{PORT}} +' > conf/config.yaml + +out=$(IP=127.0.0.1 PORT=9090 make init 2>&1 || true) +if ! echo "$out" | grep "prometheus port 9090 conflicts with control"; then + echo "failed: can't detect port conflicts" + exit 1 +fi + +echo ' +apisix: + node_listen: ${{PORT}} +plugin_attr: + prometheus: + export_addr: + ip: ${{IP}} + port: ${{PORT}} +' > conf/config.yaml + +out=$(IP=127.0.0.1 PORT=9092 make init 2>&1 || true) +if ! echo "$out" | grep "http listen port 9092 conflicts with prometheus"; then + echo "failed: can't detect port conflicts" + exit 1 +fi + +echo "passed: should detect port conflicts" + +echo ' +plugin_attr: + prometheus: + metric_prefix: apisix_ci_prefix_ + export_addr: + ip: ${{IP}} + port: ${{PORT}} +' > conf/config.yaml + +IP=127.0.0.1 PORT=9092 make run + +# initialize prometheus metrics public API route #2 +code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} -X PUT http://127.0.0.1:9180/apisix/admin/routes/metrics2 \ + -H "X-API-KEY: edd1c9f034335f136f87ad84b625c8f1" \ + -d "{ + \"uri\": \"/apisix/prometheus/metrics\", + \"plugins\": { + \"public-api\": {} + } + }") +if [ ! $code -eq 201 ]; then + echo "failed: initialize prometheus metrics public API failed #2" + exit 1 +fi + +sleep 0.5 + +if ! curl -s http://127.0.0.1:9092/apisix/prometheus/metrics | grep "apisix_ci_prefix_" | wc -l; then + echo "failed: should use custom metric prefix" + exit 1 +fi + +make stop + +echo "passed: should use custom metric prefix" From aa311b0603b6aef3591c8a786a87435942637be2 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 1 Dec 2022 18:18:39 +0800 Subject: [PATCH 10/19] debug --- .github/workflows/cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 1fcc20d5b5b4..817ed82815bb 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -31,7 +31,7 @@ jobs: - linux_apisix_current_luarocks_in_customed_nginx runs-on: ${{ matrix.platform }} - timeout-minutes: 15 + timeout-minutes: 60 env: SERVER_NAME: ${{ matrix.job_name }} OPENRESTY_VERSION: default From f05c0092aeb19d938926cfbcdccb55a073abec35 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 1 Dec 2022 18:57:34 +0800 Subject: [PATCH 11/19] fix --- .github/workflows/cli.yml | 21 +++++++++------------ t/cli/test_prometheus.sh | 2 ++ 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 817ed82815bb..2e717ae90784 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -2,15 +2,15 @@ name: CLI Test on: push: - # branches: [master, 'release/**'] - # paths-ignore: - # - 'docs/**' - # - '**/*.md' - # pull_request: - # branches: [master, 'release/**'] - # paths-ignore: - # - 'docs/**' - # - '**/*.md' + branches: [master, 'release/**'] + paths-ignore: + - 'docs/**' + - '**/*.md' + pull_request: + branches: [master, 'release/**'] + paths-ignore: + - 'docs/**' + - '**/*.md' concurrency: group: ${{ github.workflow }}-${{ github.ref == 'refs/heads/master' && github.run_number || github.ref }} @@ -62,8 +62,5 @@ jobs: sudo --preserve-env=OPENRESTY_VERSION \ ./ci/${{ matrix.job_name }}_runner.sh do_install - - name: Setup tmate session - uses: mxschmitt/action-tmate@v3 - - name: Linux Script run: sudo ./ci/${{ matrix.job_name }}_runner.sh script diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index 15f54f9114ee..2f2382980fb3 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -21,6 +21,8 @@ git checkout conf/config.yaml +sleep 1 + make run code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/prometheus/metrics) From 0663c8656d89c88a420b722fedd7f5fa250037e2 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Thu, 1 Dec 2022 18:59:16 +0800 Subject: [PATCH 12/19] revert timeout --- .github/workflows/cli.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml index 2e717ae90784..33c170af3457 100644 --- a/.github/workflows/cli.yml +++ b/.github/workflows/cli.yml @@ -31,7 +31,7 @@ jobs: - linux_apisix_current_luarocks_in_customed_nginx runs-on: ${{ matrix.platform }} - timeout-minutes: 60 + timeout-minutes: 15 env: SERVER_NAME: ${{ matrix.job_name }} OPENRESTY_VERSION: default From ab46fc12b0b8aac0f258db0c78f32f758751d7fc Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Fri, 2 Dec 2022 12:01:54 +0800 Subject: [PATCH 13/19] resolve comments --- docs/en/latest/plugins/prometheus.md | 8 +++++++- docs/zh/latest/plugins/prometheus.md | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md index 41a518ef05e0..915aa9b1f06e 100644 --- a/docs/en/latest/plugins/prometheus.md +++ b/docs/en/latest/plugins/prometheus.md @@ -102,9 +102,15 @@ plugin_attr: You can then expose it by using the [public-api](public-api.md) Plugin. -Note: If the Prometheus plugin collects too many metrics, it will take CPU resources to calculate the metric data when getting the metrics via URI, which may affect APISIX to process normal requests. To solve this problem, APISIX exposes the URI and calculates the metrics in the privileged agent. +:::info IMPORTANT + +If the Prometheus plugin collects too many metrics, it will take CPU resources to calculate the metric data when getting the metrics via URI, which may affect APISIX to process normal requests. To solve this problem, APISIX exposes the URI and calculates the metrics in the privileged agent. If the URI is exposed using the public-api plugin, then APISIX will calculate the metric data in a normal worker process, which may still affect APISIX processing of normal requests. +This feature requires APISIX to run on [APISIX-Base](../FAQ.md#how-do-i-build-the-apisix-base-environment). + +::: + ## Enabling the Plugin The `prometheus` Plugin can be enabled with an empty table. diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md index a74784427c7d..f9de82d762fd 100644 --- a/docs/zh/latest/plugins/prometheus.md +++ b/docs/zh/latest/plugins/prometheus.md @@ -85,9 +85,15 @@ plugin_attr: 你可以使用 [public-api](../../../en/latest/plugins/public-api.md) 插件来暴露该 URI。 -注意：如果 Prometheus 插件收集的指标数量过多，在通过 URI 获取指标时，会占用 CPU 资源来计算指标数据，可能会影响 APISIX 处理正常请求。为解决此问题，APISIX 在 privileged agent 中暴露 URI 并且计算指标。 +:::info IMPORTANT + +如果 Prometheus 插件收集的指标数量过多，在通过 URI 获取指标时，会占用 CPU 资源来计算指标数据，可能会影响 APISIX 处理正常请求。为解决此问题，APISIX 在 privileged agent 中暴露 URI 并且计算指标。 如果使用 public-api 插件暴露该 URI，那么 APISIX 将在普通的 worker 进程中计算指标数据，这仍可能会影响 APISIX 处理正常请求。 +该特性要求 APISIX 运行在 [APISIX-Base](../FAQ.md#如何构建-apisix-base-环境) 上。 + +::: + ## 启用插件 `prometheus` 插件可以使用空表 `{}` 开启。 From 261a0b1675468832b338abdfb43ae19da8bde1d2 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Sun, 4 Dec 2022 20:38:01 +0800 Subject: [PATCH 14/19] resolve commects --- apisix/plugins/prometheus/exporter.lua | 9 --- .../apisix/plugins/prometheus/exporter.lua | 34 +++++++++ t/cli/test_prometheus.sh | 2 - t/cli/test_prometheus_run_in_privileged.sh | 76 ++++++++++++++++++- t/cli/test_prometheus_stream.sh | 22 ------ 5 files changed, 107 insertions(+), 36 deletions(-) create mode 100644 example/apisix/plugins/prometheus/exporter.lua diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua index be466525a0ef..cd8dc1bf0934 100644 --- a/apisix/plugins/prometheus/exporter.lua +++ b/apisix/plugins/prometheus/exporter.lua @@ -411,16 +411,7 @@ local function shared_dict_status() end --- for test -local function log_process_type() - local process_type = require("ngx.process").type() - return process_type -end - - local function collect(ctx, stream_only) - core.log.info("process type: ", core.log.delay_exec(log_process_type)) - if not prometheus or not metrics then core.log.error("prometheus: plugin is not initialized, please make sure ", " 'prometheus_metrics' shared dict is present in nginx template") diff --git a/example/apisix/plugins/prometheus/exporter.lua b/example/apisix/plugins/prometheus/exporter.lua new file mode 100644 index 000000000000..982b00f044ca --- /dev/null +++ b/example/apisix/plugins/prometheus/exporter.lua @@ -0,0 +1,34 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local apisix = require("apisix") +local core = require("apisix.core") +local _M = {} + + +function _M.http_init() + return true +end + + +function _M.export_metrics() + local process_type = require("ngx.process").type() + core.log.info("process type: ", process_type) + return core.response.exit(200) +end + + +return _M diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index 2f2382980fb3..15f54f9114ee 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -21,8 +21,6 @@ git checkout conf/config.yaml -sleep 1 - make run code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/prometheus/metrics) diff --git a/t/cli/test_prometheus_run_in_privileged.sh b/t/cli/test_prometheus_run_in_privileged.sh index 8b03566862bc..85fab2e57aca 100755 --- a/t/cli/test_prometheus_run_in_privileged.sh +++ b/t/cli/test_prometheus_run_in_privileged.sh @@ -23,16 +23,20 @@ git checkout conf/config.yaml exit_if_not_customed_nginx -# collect metrics run in privileged agent +# prometheus run in privileged works when only http is enabled +sleep 0.5 rm logs/error.log || true echo ' +apisix: + extra_lua_path: "\$prefix/example/?.lua" + lua_module_hook: "apisix.plugins.prometheus.exporter" nginx_config: - error_log_level: info + error_log_level: info ' > conf/config.yaml -make init make run +sleep 0.1 curl -s -o /dev/null http://127.0.0.1:9091/apisix/prometheus/metrics @@ -41,4 +45,70 @@ if ! grep -E "process type: privileged agent" logs/error.log; then exit 1 fi +make stop + echo "prometheus works well in privileged agent successfully" + + +# prometheus run in privileged works when both http & stream are enabled +sleep 0.5 +rm logs/error.log || true + +echo " +apisix: + extra_lua_path: "\$prefix/example/?.lua" + lua_module_hook: "apisix.plugins.prometheus.exporter" + enable_admin: true + stream_proxy: + tcp: + - addr: 9100 +stream_plugins: + - prometheus +nginx_config: + error_log_level: info +" > conf/config.yaml + +make run +sleep 0.1 + +curl -s -o /dev/null http://127.0.0.1:9091/apisix/prometheus/metrics + +if ! grep -E " process type: privileged agent" logs/error.log; then + echo "failed: prometheus run in privileged can't work when both http & stream are enabled" + exit 1 +fi + +echo "passed: prometheus run in privileged works when both http & stream are enabled" + +make stop + + +# prometheus run in privileged works when only stream is enabled +sleep 0.5 +rm logs/error.log || true + +echo " +apisix: + extra_lua_path: "\$prefix/example/?.lua" + lua_module_hook: "apisix.plugins.prometheus.exporter" + enable_admin: false + stream_proxy: + tcp: + - addr: 9100 +stream_plugins: + - prometheus +nginx_config: + error_log_level: info +" > conf/config.yaml + +make run +sleep 0.1 + +curl -s -o /dev/null http://127.0.0.1:9091/apisix/prometheus/metrics + +if ! grep -E " process type: privileged agent" logs/error.log; then + echo "failed: prometheus run in privileged can't work when only stream is enabled" + exit 1 +fi + +echo "passed: prometheus run in privileged works when only stream is enabled" diff --git a/t/cli/test_prometheus_stream.sh b/t/cli/test_prometheus_stream.sh index 6f699f19e619..561b9a820cf5 100755 --- a/t/cli/test_prometheus_stream.sh +++ b/t/cli/test_prometheus_stream.sh @@ -21,8 +21,6 @@ exit_if_not_customed_nginx -rm logs/error.log || true - echo " apisix: enable_admin: true @@ -31,8 +29,6 @@ apisix: - addr: 9100 stream_plugins: - prometheus -nginx_config: - error_log_level: info " > conf/config.yaml make run @@ -67,15 +63,6 @@ make stop echo "passed: prometheus works when both http & stream are enabled" -if ! grep -E " process type: privileged agent" logs/error.log; then - echo "failed: prometheus run in privileged can't work when both http & stream are enabled" - exit 1 -fi - -echo "passed: prometheus run in privileged works when both http & stream are enabled" - -rm logs/error.log || true - echo " apisix: enable_admin: false @@ -84,8 +71,6 @@ apisix: - addr: 9100 stream_plugins: - prometheus -nginx_config: - error_log_level: info " > conf/config.yaml make run @@ -106,10 +91,3 @@ if ! echo "$out" | grep "apisix_node_info{hostname=" > /dev/null; then fi echo "passed: prometheus works when only stream is enabled" - -if ! grep -E " process type: privileged agent" logs/error.log; then - echo "failed: prometheus run in privileged can't work when only stream is enabled" - exit 1 -fi - -echo "passed: prometheus run in privileged works when only stream is enabled" From 011ab0cdb34da94f9f68f90e5292d49adcee1778 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Sun, 4 Dec 2022 20:39:23 +0800 Subject: [PATCH 15/19] revert exporter --- apisix/plugins/prometheus/exporter.lua | 1 - 1 file changed, 1 deletion(-) diff --git a/apisix/plugins/prometheus/exporter.lua b/apisix/plugins/prometheus/exporter.lua index cd8dc1bf0934..45ff94c3f631 100644 --- a/apisix/plugins/prometheus/exporter.lua +++ b/apisix/plugins/prometheus/exporter.lua @@ -14,7 +14,6 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- -local require = require local base_prometheus = require("prometheus") local core = require("apisix.core") local plugin = require("apisix.plugin") From 1814f4e5fff06ea317af4e3cca48351e1c0683a3 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Sun, 4 Dec 2022 20:42:42 +0800 Subject: [PATCH 16/19] chore echo --- t/cli/test_prometheus_run_in_privileged.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/t/cli/test_prometheus_run_in_privileged.sh b/t/cli/test_prometheus_run_in_privileged.sh index 85fab2e57aca..fdf65de2b991 100755 --- a/t/cli/test_prometheus_run_in_privileged.sh +++ b/t/cli/test_prometheus_run_in_privileged.sh @@ -41,13 +41,13 @@ sleep 0.1 curl -s -o /dev/null http://127.0.0.1:9091/apisix/prometheus/metrics if ! grep -E "process type: privileged agent" logs/error.log; then - echo "failed: prometheus works well in privileged agent" + echo "failed: prometheus run in privileged can't work when only http is enabled" exit 1 fi make stop -echo "prometheus works well in privileged agent successfully" +echo "prometheus run in privileged agent successfully when only http is enabled" # prometheus run in privileged works when both http & stream are enabled @@ -78,7 +78,7 @@ if ! grep -E " process type: privileged agent" logs/error.log; then exit 1 fi -echo "passed: prometheus run in privileged works when both http & stream are enabled" +echo "passed: prometheus run in privileged agent successfully when both http & stream are enabled" make stop @@ -111,4 +111,4 @@ if ! grep -E " process type: privileged agent" logs/error.log; then exit 1 fi -echo "passed: prometheus run in privileged works when only stream is enabled" +echo "passed: prometheus run in privileged agent successfully when only stream is enabled" From 4c1b16290d81911d77af9a4d55feda14681991b5 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Sun, 4 Dec 2022 21:06:30 +0800 Subject: [PATCH 17/19] fix --- t/cli/test_prometheus.sh | 2 ++ t/cli/test_prometheus_run_in_privileged.sh | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/t/cli/test_prometheus.sh b/t/cli/test_prometheus.sh index 15f54f9114ee..2f2382980fb3 100755 --- a/t/cli/test_prometheus.sh +++ b/t/cli/test_prometheus.sh @@ -21,6 +21,8 @@ git checkout conf/config.yaml +sleep 1 + make run code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/prometheus/metrics) diff --git a/t/cli/test_prometheus_run_in_privileged.sh b/t/cli/test_prometheus_run_in_privileged.sh index fdf65de2b991..295743ca704a 100755 --- a/t/cli/test_prometheus_run_in_privileged.sh +++ b/t/cli/test_prometheus_run_in_privileged.sh @@ -30,7 +30,7 @@ rm logs/error.log || true echo ' apisix: extra_lua_path: "\$prefix/example/?.lua" - lua_module_hook: "apisix.plugins.prometheus.exporter" + lua_module_hook: apisix.plugins.prometheus.exporter nginx_config: error_log_level: info ' > conf/config.yaml @@ -57,7 +57,7 @@ rm logs/error.log || true echo " apisix: extra_lua_path: "\$prefix/example/?.lua" - lua_module_hook: "apisix.plugins.prometheus.exporter" + lua_module_hook: apisix.plugins.prometheus.exporter enable_admin: true stream_proxy: tcp: @@ -90,7 +90,7 @@ rm logs/error.log || true echo " apisix: extra_lua_path: "\$prefix/example/?.lua" - lua_module_hook: "apisix.plugins.prometheus.exporter" + lua_module_hook: apisix.plugins.prometheus.exporter enable_admin: false stream_proxy: tcp: From bf4d8d435fc59fd3eb4484533cbd6c2ae2863519 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 5 Dec 2022 10:28:30 +0800 Subject: [PATCH 18/19] resolve comments --- docs/en/latest/plugins/prometheus.md | 2 +- docs/zh/latest/plugins/prometheus.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/latest/plugins/prometheus.md b/docs/en/latest/plugins/prometheus.md index 915aa9b1f06e..4b6e41d8ead7 100644 --- a/docs/en/latest/plugins/prometheus.md +++ b/docs/en/latest/plugins/prometheus.md @@ -104,7 +104,7 @@ You can then expose it by using the [public-api](public-api.md) Plugin. :::info IMPORTANT -If the Prometheus plugin collects too many metrics, it will take CPU resources to calculate the metric data when getting the metrics via URI, which may affect APISIX to process normal requests. To solve this problem, APISIX exposes the URI and calculates the metrics in the privileged agent. +If the Prometheus plugin collects too many metrics, it will take CPU resources to calculate the metric data when getting the metrics via URI, which may affect APISIX to process normal requests. To solve this problem, APISIX exposes the URI and calculates the metrics in the [privileged agent](https://github.com/openresty/lua-resty-core/blob/master/lib/ngx/process.md#enable_privileged_agent). If the URI is exposed using the public-api plugin, then APISIX will calculate the metric data in a normal worker process, which may still affect APISIX processing of normal requests. This feature requires APISIX to run on [APISIX-Base](../FAQ.md#how-do-i-build-the-apisix-base-environment). diff --git a/docs/zh/latest/plugins/prometheus.md b/docs/zh/latest/plugins/prometheus.md index f9de82d762fd..ce59b87ec11b 100644 --- a/docs/zh/latest/plugins/prometheus.md +++ b/docs/zh/latest/plugins/prometheus.md @@ -87,7 +87,7 @@ plugin_attr: :::info IMPORTANT -如果 Prometheus 插件收集的指标数量过多，在通过 URI 获取指标时，会占用 CPU 资源来计算指标数据，可能会影响 APISIX 处理正常请求。为解决此问题，APISIX 在 privileged agent 中暴露 URI 并且计算指标。 +如果 Prometheus 插件收集的指标数量过多，在通过 URI 获取指标时，会占用 CPU 资源来计算指标数据，可能会影响 APISIX 处理正常请求。为解决此问题，APISIX 在 [privileged agent](https://github.com/openresty/lua-resty-core/blob/master/lib/ngx/process.md#enable_privileged_agent) 中暴露 URI 并且计算指标。 如果使用 public-api 插件暴露该 URI，那么 APISIX 将在普通的 worker 进程中计算指标数据，这仍可能会影响 APISIX 处理正常请求。 该特性要求 APISIX 运行在 [APISIX-Base](../FAQ.md#如何构建-apisix-base-环境) 上。 From 9e8b135209385c1bb1c52973eff5d0ea029e7675 Mon Sep 17 00:00:00 2001 From: tzssangglass Date: Mon, 5 Dec 2022 14:26:54 +0800 Subject: [PATCH 19/19] move exporter --- apisix/cli/ngx_tpl.lua | 1 - t/cli/test_prometheus_run_in_privileged.sh | 9 +++------ .../lib}/apisix/plugins/prometheus/exporter.lua | 7 ++++++- 3 files changed, 9 insertions(+), 8 deletions(-) rename {example => t/lib}/apisix/plugins/prometheus/exporter.lua (95%) diff --git a/apisix/cli/ngx_tpl.lua b/apisix/cli/ngx_tpl.lua index 8e4263081c3e..b578ec27ffd5 100644 --- a/apisix/cli/ngx_tpl.lua +++ b/apisix/cli/ngx_tpl.lua @@ -78,7 +78,6 @@ http { init_by_lua_block { require "resty.core" - apisix = require("apisix") local process = require("ngx.process") local ok, err = process.enable_privileged_agent() if not ok then diff --git a/t/cli/test_prometheus_run_in_privileged.sh b/t/cli/test_prometheus_run_in_privileged.sh index 295743ca704a..7f8a3e2ec5ff 100755 --- a/t/cli/test_prometheus_run_in_privileged.sh +++ b/t/cli/test_prometheus_run_in_privileged.sh @@ -29,8 +29,7 @@ rm logs/error.log || true echo ' apisix: - extra_lua_path: "\$prefix/example/?.lua" - lua_module_hook: apisix.plugins.prometheus.exporter + extra_lua_path: "\$prefix/t/lib/?.lua" nginx_config: error_log_level: info ' > conf/config.yaml @@ -56,8 +55,7 @@ rm logs/error.log || true echo " apisix: - extra_lua_path: "\$prefix/example/?.lua" - lua_module_hook: apisix.plugins.prometheus.exporter + extra_lua_path: "\$prefix/t/lib/?.lua" enable_admin: true stream_proxy: tcp: @@ -89,8 +87,7 @@ rm logs/error.log || true echo " apisix: - extra_lua_path: "\$prefix/example/?.lua" - lua_module_hook: apisix.plugins.prometheus.exporter + extra_lua_path: "\$prefix/t/lib/?.lua" enable_admin: false stream_proxy: tcp: diff --git a/example/apisix/plugins/prometheus/exporter.lua b/t/lib/apisix/plugins/prometheus/exporter.lua similarity index 95% rename from example/apisix/plugins/prometheus/exporter.lua rename to t/lib/apisix/plugins/prometheus/exporter.lua index 982b00f044ca..c9c71f606dd1 100644 --- a/example/apisix/plugins/prometheus/exporter.lua +++ b/t/lib/apisix/plugins/prometheus/exporter.lua @@ -14,7 +14,7 @@ -- See the License for the specific language governing permissions and -- limitations under the License. -- -local apisix = require("apisix") + local core = require("apisix.core") local _M = {} @@ -24,6 +24,11 @@ function _M.http_init() end +function _M.stream_init() + return true +end + + function _M.export_metrics() local process_type = require("ngx.process").type() core.log.info("process type: ", process_type)