change(prometheus): don't expose metrics to internet

apisix/cli/ngx_tpl.lua

@@ -288,6 +288,21 @@ http {
     }
     {% end %}
 
+    {% if enabled_plugins["prometheus"] and prometheus_server_addr then %}
+    server {
+        listen {* prometheus_server_addr *};
+
+        access_log off;
+
+        location / {
+            content_by_lua_block {
+                local prometheus = require("apisix.plugins.prometheus")
+                prometheus.export_metrics()
+            }
+        }
+    }
+    {% end %}
+
     {% if enable_admin and port_admin then %}
     server {
         {%if https_admin then%}

apisix/cli/ops.lua

@@ -494,6 +494,24 @@ Please modify "admin_key" in conf/config.yaml .
         end
     end
 
+    if yaml_conf.plugin_attr.prometheus then
+        local prometheus = yaml_conf.plugin_attr.prometheus
+        if prometheus.enable_export_server then
+            local ip = prometheus.export_addr.ip
+            local port = tonumber(prometheus.export_addr.port)
+
+            if ip == nil then
+                ip = "127.0.0.1"
+            end
+
+            if not port then
+                port = 9091
+            end
+
+            sys_conf.prometheus_server_addr = ip .. ":" .. port
+        end
+    end
+
     local wrn = sys_conf["worker_rlimit_nofile"]
     local wc = sys_conf["event"]["worker_connections"]
     if not wrn or wrn <= wc then

apisix/plugins/prometheus.lua

@@ -14,6 +14,7 @@
 -- See the License for the specific language governing permissions and
 -- limitations under the License.
 --
+local ngx = ngx
 local core = require("apisix.core")
 local exporter = require("apisix.plugins.prometheus.exporter")
 
@@ -45,7 +46,7 @@ function _M.check_schema(conf)
 end
 
 
-function _M.api()
+local function get_api(called_by_api_router)
     local export_uri = default_export_uri
     local local_conf = core.config.local_conf()
     local attr = core.table.try_read_attr(local_conf, "plugin_attr",
@@ -54,13 +55,39 @@ function _M.api()
         export_uri = attr.export_uri
     end
 
-    return {
-        {
-            methods = {"GET"},
-            uri = export_uri,
-            handler = exporter.collect
-        }
+    local api = {
+        methods = {"GET"},
+        uri = export_uri,
+        handler = exporter.collect
     }
+
+    if not called_by_api_router then
+        return api
+    end
+
+    if attr.enable_export_server then
+        return {}
+    end
+
+    return {api}
+end
+
+
+function _M.api()
+    return get_api(true)
+end
+
+
+function _M.export_metrics()
+    local api = get_api(false)
+    local uri = ngx.var.uri
+    local method = ngx.req.get_method()
+
+    if uri == api.uri and method == api.methods[1] then
+        return api.handler()
+    end
+
+    return core.response.exit(404)
 end
 
 

conf/config-default.yaml

@@ -288,6 +288,10 @@ plugin_attr:
     endpoint_addr: http://127.0.0.1:12800
   prometheus:
     export_uri: /apisix/prometheus/metrics
+    enable_export_server: true
+    export_addr:
+      ip: "127.0.0.1"
+      port: 9091
   server-info:
     report_interval: 60  # server info report interval (unit: second)
     report_ttl: 3600     # live time for server info in etcd (unit: second)

docs/en/latest/plugins/prometheus.md

@@ -30,7 +30,30 @@ none.
 ## API
 
 This plugin will add `/apisix/prometheus/metrics` to expose the metrics.
-You may need to use [interceptors](../plugin-interceptors.md) to protect it.
+
+The metrics are exposed via a separate Prometheus server address.
+By default, the address is `127.0.0.1:9091`. You can change it in the `conf/config.yaml`, for example:
+
+```
+plugin_attr:
+  prometheus:
+    export_addr:
+      ip: ${{INTRANET_IP}}
+      port: 9092
+```
+
+Assume environment variable `INTRANET_IP` is `172.1.1.1`, now APISIX will export the metrics via `172.1.1.1:9092`.
+
+**Before version `2.6`, the metrics are exposed via the data panel port,
+you may need to use [interceptors](../plugin-interceptors.md) to protect it.**
+
+If you still want this behavior, you can configure it like this:
+
+```
+plugin_attr:
+  prometheus:
+    enable_export_server: false
+```
 
 ## How to enable it
 
@@ -68,7 +91,7 @@ Then add prometheus plugin:
 We fetch the metric data from the specified url `/apisix/prometheus/metrics`.
 
 ```
-curl -i http://127.0.0.1:9080/apisix/prometheus/metrics
+curl -i http://127.0.0.1:9091/apisix/prometheus/metrics
 ```
 
 Puts this URL address into prometheus, and it will automatically fetch
@@ -81,7 +104,7 @@ scrape_configs:
   - job_name: 'apisix'
     metrics_path: '/apisix/prometheus/metrics'
     static_configs:
-    - targets: ['127.0.0.1:9080']
+    - targets: ['127.0.0.1:9091']
 ```
 
 And we can check the status at prometheus console:

docs/zh/latest/plugins/prometheus.md

@@ -29,8 +29,31 @@ title: prometheus
 
 ## 接口
 
-插件会增加 `/apisix/prometheus/metrics` 这个接口，你可能需要通过 [interceptors](../plugin-interceptors.md)
-来保护它。
+插件会增加 `/apisix/prometheus/metrics` 这个接口。
+
+指标默认会通过独立的服务地址暴露。
+默认情况下，这个地址是 `127.0.0.1:9091`。你可以在 `conf/config.yaml` 里面修改它，比如：
+
+```
+plugin_attr:
+  prometheus:
+    export_addr:
+      ip: ${{INTRANET_IP}}
+      port: 9092
+```
+
+假设环境变量 `INTRANET_IP` 是 `172.1.1.1`，现在 APISIX 会在 `172.1.1.1:9092` 上暴露指标。
+
+**在 2.6 版本之前，指标会直接暴露到数据面的端口上，你可能需要通过 [interceptors](../plugin-interceptors.md)
+来保护它。**
+
+如果你依然想要这样的行为，你可以这么配置：
+
+```
+plugin_attr:
+  prometheus:
+    enable_export_server: false
+```
 
 ## 如何开启插件
 
@@ -67,7 +90,7 @@ curl http://127.0.0.1:9080/apisix/admin/routes/1 -H 'X-API-KEY: edd1c9f034335f13
 我们可以从指定的 url 中提取指标数据 `/apisix/prometheus/metrics`:
 
 ```
-curl -i http://127.0.0.1:9080/apisix/prometheus/metrics
+curl -i http://127.0.0.1:9091/apisix/prometheus/metrics
 ```
 
 把该 uri 地址配置到 prometheus 中去,就会自动完成指标数据提取.
@@ -79,7 +102,7 @@ scrape_configs:
   - job_name: "apisix"
     metrics_path: "/apisix/prometheus/metrics"
     static_configs:
-      - targets: ["127.0.0.1:9080"]
+      - targets: ["127.0.0.1:9091"]
 ```
 
 我们也可以在 prometheus 控制台中去检查状态:

t/APISIX.pm

@@ -79,6 +79,7 @@ if ($custom_dns_server) {
 my $default_yaml_config = read_file("conf/config-default.yaml");
 # enable example-plugin as some tests require it
 $default_yaml_config =~ s/# - example-plugin/- example-plugin/;
+$default_yaml_config =~ s/enable_export_server: true/enable_export_server: false/;
 
 my $user_yaml_config = read_file("conf/config.yaml");
 my $ssl_crt = read_file("t/certs/apisix.crt");

t/chaos/setup_chaos_utils.sh

@@ -35,7 +35,11 @@ modify_config() {
   - ${DNS_IP}
 etcd:
   host:
-    - \"http://etcd-cluster-client.default.svc.cluster.local:2379\" " > ./conf/config.yaml
+  - \"http://etcd-cluster-client.default.svc.cluster.local:2379\"
+plugin_attr:
+  prometheus:
+    enable_export_server: false
+  " > ./conf/config.yaml
     sed -i -e 's/apisix:latest/apisix:alpine-local/g' kubernetes/deployment.yaml
 }
 

t/cli/test_access_log.sh

@@ -57,7 +57,7 @@ if [ $count_test_access_log -eq 0 ]; then
 fi
 
 count_access_log_off=`grep -c "access_log off;" conf/nginx.conf || true`
-if [ $count_access_log_off -eq 3 ]; then
+if [ $count_access_log_off -eq 4 ]; then
     echo "failed: nginx.conf file find access_log off; when enable access log"
     exit 1
 fi
@@ -92,7 +92,7 @@ if [ $count_test_access_log -eq 1 ]; then
 fi
 
 count_access_log_off=`grep -c "access_log off;" conf/nginx.conf || true`
-if [ $count_access_log_off -ne 3 ]; then
+if [ $count_access_log_off -ne 4 ]; then
     echo "failed: nginx.conf file doesn't find access_log off; when disable access log"
     exit 1
 fi

t/cli/test_prometheus.sh

@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+. ./t/cli/common.sh
+
+git checkout conf/config.yaml
+
+make run
+
+code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/prometheus/metrics)
+if [ ! $code -eq 404 ]; then
+    echo "failed: should listen default prometheus address"
+    exit 1
+fi
+
+code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9091/apisix/prometheus/metrics)
+if [ ! $code -eq 200 ]; then
+    echo "failed: should listen default prometheus address"
+    exit 1
+fi
+
+make stop
+
+echo "passed: should listen default prometheus address"
+
+echo '
+plugin_attr:
+  prometheus:
+    export_addr:
+        ip: ${{IP}}
+        port: ${{PORT}}
+' > conf/config.yaml
+
+IP=127.0.0.1 PORT=9092 make run
+
+code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9092/apisix/prometheus/metrics)
+if [ ! $code -eq 200 ]; then
+    echo "failed: should listen configured prometheus address"
+    exit 1
+fi
+
+make stop
+
+echo "passed: should listen configured prometheus address"
+
+echo '
+plugin_attr:
+  prometheus:
+    enable_export_server: false
+    export_uri: /prometheus/metrics
+    export_addr:
+        ip: ${{IP}}
+        port: ${{PORT}}
+' > conf/config.yaml
+
+IP=127.0.0.1 PORT=9092 make run
+
+code=$(curl -v -k -i -m 20 -o /dev/null -s http://127.0.0.1:9092/prometheus/metrics || echo 'ouch')
+if [ "$code" != "ouch" ]; then
+    echo "failed: should listen previous prometheus address"
+    exit 1
+fi
+
+code=$(curl -v -k -i -m 20 -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/prometheus/metrics)
+if [ ! $code -eq 200 ]; then
+    echo "failed: should listen previous prometheus address"
+    exit 1
+fi
+
+make stop
+
+echo "passed: should listen previous prometheus address"