bugfix ^o^ token error, protect metrics api auth (#1434)

Signed-off-by: tomsun28 <tomsun28@outlook.com>
apache · Mar 10, 2024 · bb560aa · bb560aa
1 parent fcb84e9
commit bb560aa
Show file tree

Hide file tree

Showing 19 changed files with 129 additions and 79 deletions.
diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/Collector.java b/collector/src/main/java/org/dromara/hertzbeat/collector/Collector.java
@@ -3,6 +3,8 @@
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
 
+import javax.annotation.PostConstruct;
+
 /**
  * collector startup
  * @author tom
@@ -12,4 +14,9 @@ public class Collector {
     public static void main(String[] args) {
         SpringApplication.run(Collector.class, args);
     }
+
+    @PostConstruct
+    public void init() {
+        System.setProperty("jdk.jndi.object.factoriesFilter", "!com.zaxxer.hikari.HikariJNDIFactory");
+    }
 }
diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/CommonDispatcher.java b/collector/src/main/java/org/dromara/hertzbeat/collector/dispatch/CommonDispatcher.java
@@ -62,6 +62,10 @@ public class CommonDispatcher implements MetricsTaskDispatch, CollectDataDispatc
      * Trigger sub task max num
      */
     private static final int MAX_SUB_TASK_NUM = 50;
+    /**
+     * Collect Response env config length
+     */
+    private static final int ENV_CONFIG_SIZE = 1;
     private static final Gson GSON = new Gson();
     /**
      * Priority queue of index collection tasks
@@ -256,21 +260,29 @@ public void dispatchCollectData(Timeout timeout, Metrics metrics, CollectRep.Met
                 // The execution of the current level metrics is completed, and the execution of the next level metrics starts
                 // use pre collect metrics data to replace next metrics config params
                 List<Map<String, Configmap>> configmapList = getConfigmapFromPreCollectData(metricsData);
+                if (configmapList.size() == ENV_CONFIG_SIZE) {
+                    job.addEnvConfigmaps(configmapList.get(0));
+                }
                 for (Metrics metricItem : metricsSet) {
-                    if (CollectionUtils.isEmpty(configmapList) || CollectUtil.notContainCryPlaceholder(GSON.toJsonTree(metricItem))) {
-                        MetricsCollect metricsCollect = new MetricsCollect(metricItem, timeout, this, 
+                    Set<String> cryPlaceholderFields = CollectUtil.matchCryPlaceholderField(GSON.toJsonTree(metricItem));
+                    if (cryPlaceholderFields.isEmpty()) {
+                        MetricsCollect metricsCollect = new MetricsCollect(metricItem, timeout, this,
                                 collectorIdentity, unitConvertList);
                         jobRequestQueue.addJob(metricsCollect);
                         metricsTimeoutMonitorMap.put(job.getId() + "-" + metricItem.getName(),
                                 new MetricsTime(System.currentTimeMillis(), metricItem, timeout));
                         continue;
                     }
-
-                    int subTaskNum = Math.min(configmapList.size(), MAX_SUB_TASK_NUM);
+                    boolean isSubTask = configmapList.stream().anyMatch(map -> map.keySet().stream().anyMatch(cryPlaceholderFields::contains));
+                    int subTaskNum = isSubTask ? Math.min(configmapList.size(), MAX_SUB_TASK_NUM) : 1;
                     AtomicInteger subTaskNumAtomic = new AtomicInteger(subTaskNum);
                     AtomicReference<CollectRep.MetricsData> metricsDataReference = new AtomicReference<>();
                     for (int index = 0; index < subTaskNum; index++) {
-                        Map<String, Configmap> configmap = configmapList.get(index);
+                        Map<String, Configmap> configmap = new HashMap<>(job.getEnvConfigmaps());
+                        if (isSubTask) {
+                            Map<String, Configmap> preConfigMap = configmapList.get(index);
+                            configmap.putAll(preConfigMap);
+                        }
                         JsonElement metricJson = GSON.toJsonTree(metricItem);
                         CollectUtil.replaceCryPlaceholder(metricJson, configmap);
                         Metrics metric = GSON.fromJson(metricJson, Metrics.class);
@@ -351,7 +363,7 @@ public void dispatchCollectData(Timeout timeout, Metrics metrics, List<CollectRe
 
     private List<Map<String, Configmap>> getConfigmapFromPreCollectData(CollectRep.MetricsData metricsData) {
         if (metricsData.getValuesCount() <= 0 || metricsData.getFieldsCount() <= 0) {
-            return null;
+            return new LinkedList<>();
         }
         List<Map<String, Configmap>> mapList = new LinkedList<>();
         for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) {

diff --git a/collector/src/main/java/org/dromara/hertzbeat/collector/util/CollectUtil.java b/collector/src/main/java/org/dromara/hertzbeat/collector/util/CollectUtil.java
@@ -25,12 +25,10 @@
 import org.dromara.hertzbeat.common.entity.job.Metrics;
 import org.dromara.hertzbeat.common.util.JsonUtil;
 
-import java.util.Arrays;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
 
 /**
  * util for collector
@@ -177,7 +175,7 @@ public static Boolean assertPromRequireField(String aliasField) {
 
 
     /**
-     * is contains cryPlaceholder -_-
+     * is contains cryPlaceholder ^o^xxx^o^
      *
      * @param jsonElement json element
      * @return return true when contains
@@ -191,6 +189,18 @@ public static boolean notContainCryPlaceholder(JsonElement jsonElement) {
         return !containCryPlaceholder(jsonElement);
     }
 
+    /**
+     * match existed cry placeholder fields ^o^field^o^
+     * @param jsonElement json element
+     * @return match field str
+     */
+    public static Set<String> matchCryPlaceholderField(JsonElement jsonElement) {
+        String jsonStr = jsonElement.toString();
+        return CRYING_PLACEHOLDER_REGEX_PATTERN.matcher(jsonStr).results()
+                .map(item -> item.group().replaceAll(CRYING_PLACEHOLDER_REX, ""))
+                .collect(Collectors.toSet());
+    }
+
     /**
      * json parameter replacement
      *

diff --git a/common/src/main/java/org/dromara/hertzbeat/common/config/AviatorConfiguration.java b/common/src/main/java/org/dromara/hertzbeat/common/config/AviatorConfiguration.java
@@ -52,7 +52,10 @@ public void configAviatorEvaluator() {
                 .useLRUExpressionCache(AVIATOR_LRU_CACHE_SIZE)
                 .addFunction(new StrEqualFunction());
 
-        // 配置Aviator语法特性集合
+        // limit loop 限制循环次数
+        instance.setOption(Options.MAX_LOOP_COUNT, 10);
+
+        // 启用部分Aviator语法特性集合
         instance.setOption(Options.FEATURE_SET,
                 Feature.asSet(Feature.If,
                         Feature.Assignment,

diff --git a/common/src/main/java/org/dromara/hertzbeat/common/entity/job/Job.java b/common/src/main/java/org/dromara/hertzbeat/common/entity/job/Job.java
@@ -109,6 +109,13 @@ public class Job {
      */
     private List<Configmap> configmap;
 
+    /**
+     * the collect data response metrics as env configmap for other collect use. ^o^xxx^o^
+     * 优先级高的采集响应单行指标可以作为后续优先级采集配置的环境变量 ^o^xxx^o^
+     */
+    @JsonIgnore
+    private Map<String, Configmap> envConfigmaps;
+
     /**
      * collector use - timestamp when the task was scheduled by the time wheel
      * 任务被时间轮开始调度的时间戳
@@ -170,6 +177,7 @@ public synchronized void constructPriorMetrics() {
                 return Byte.MAX_VALUE;
             }
         }));
+        envConfigmaps = new HashMap<>(8);
     }
 
     /**
@@ -225,6 +233,18 @@ public void addCollectMetricsData(CollectRep.MetricsData metricsData) {
         responseDataTemp.add(metricsData);
     }
 
+    public Map<String, Configmap> getEnvConfigmaps() {
+        return envConfigmaps;
+    }
+
+    public void addEnvConfigmaps(Map<String, Configmap> envConfigmaps) {
+        if (this.envConfigmaps == null) {
+            this.envConfigmaps = envConfigmaps;
+        } else {
+            this.envConfigmaps.putAll(envConfigmaps);   
+        }
+    }
+
     @Override
     public Job clone() {
         // deep clone   深度克隆

diff --git a/manager/src/main/resources/application.yml b/manager/src/main/resources/application.yml
@@ -44,7 +44,6 @@ management:
         include:
           - 'metrics'
           - 'health'
-          - 'env'
     enabled-by-default: on
 
 sureness:

diff --git a/manager/src/main/resources/define/app-hertzbeat.yml b/manager/src/main/resources/define/app-hertzbeat.yml
@@ -170,6 +170,12 @@ metrics:
       timeout: ^_^timeout^_^
       method: GET
       ssl: ^_^ssl^_^
+      authorization:
+        type: ^_^authType^_^
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+        digestAuthUsername: ^_^username^_^
+        digestAuthPassword: ^_^password^_^
       parseType: jsonPath
       parseScript: '$.data'
 
@@ -181,23 +187,17 @@ metrics:
         type: 1
     protocol: http
     http:
-      # http host: ipv4 ipv6 domain
-      # 主机host: ipv4 ipv6 域名
       host: ^_^host^_^
-      # http port
-      # 端口
       port: ^_^port^_^
-      # http url
-      # url请求接口路径
       url: /actuator/metrics/jvm.threads.states
-      # http method: GET POST PUT DELETE PATCH
-      # 请求方式 GET POST PUT DELETE PATCH
       method: GET
-      # if enabled https, default value is false
-      # 是否启用ssl/tls,即是http还是https,默认false
       ssl: ^_^ssl^_^
-      # http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring
-      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控
+      authorization:
+        type: ^_^authType^_^
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+        digestAuthUsername: ^_^username^_^
+        digestAuthPassword: ^_^password^_^
       parseType: jsonPath
       parseScript: '$.availableTags[?(@.tag == "state")].values[*]'
 
@@ -215,23 +215,17 @@ metrics:
       - number=#`$.measurements[?(@.statistic == "VALUE")].value`
     protocol: http
     http:
-      # http host: ipv4 ipv6 domain
-      # 主机host: ipv4 ipv6 域名
       host: ^_^host^_^
-      # http port
-      # 端口
       port: ^_^port^_^
-      # http url
-      # url请求接口路径
       url: /actuator/metrics/jvm.threads.states?tag=state:^o^state^o^
-      # http method: GET POST PUT DELETE PATCH
-      # 请求方式 GET POST PUT DELETE PATCH
       method: GET
-      # if enabled https, default value is false
-      # 是否启用ssl/tls,即是http还是https,默认false
       ssl: ^_^ssl^_^
-      # http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring
-      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控
+      authorization:
+        type: ^_^authType^_^
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+        digestAuthUsername: ^_^username^_^
+        digestAuthPassword: ^_^password^_^
       parseType: jsonPath
       parseScript: '$'
 
@@ -243,23 +237,17 @@ metrics:
         type: 1
     protocol: http
     http:
-      # http host: ipv4 ipv6 domain
-      # 主机host: ipv4 ipv6 域名
       host: ^_^host^_^
-      # http port
-      # 端口
       port: ^_^port^_^
-      # http url
-      # url请求接口路径
       url: /actuator/metrics/jvm.memory.used
-      # http method: GET POST PUT DELETE PATCH
-      # 请求方式 GET POST PUT DELETE PATCH
       method: GET
-      # if enabled https, default value is false
-      # 是否启用ssl/tls,即是http还是https,默认false
       ssl: ^_^ssl^_^
-      # http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring
-      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控
+      authorization:
+        type: ^_^authType^_^
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+        digestAuthUsername: ^_^username^_^
+        digestAuthPassword: ^_^password^_^
       parseType: jsonPath
       parseScript: '$.availableTags[?(@.tag == "id")].values[*]'
 
@@ -280,22 +268,16 @@ metrics:
       - mem_used=B->MB
     protocol: http
     http:
-      # http host: ipv4 ipv6 domain
-      # 主机host: ipv4 ipv6 域名
       host: ^_^host^_^
-      # http port
-      # 端口
       port: ^_^port^_^
-      # http url
-      # url请求接口路径
       url: /actuator/metrics/jvm.memory.used?tag=id:^o^id^o^
-      # http method: GET POST PUT DELETE PATCH
-      # 请求方式 GET POST PUT DELETE PATCH
       method: GET
-      # if enabled https, default value is false
-      # 是否启用ssl/tls,即是http还是https,默认false
       ssl: ^_^ssl^_^
-      # http response data parse type: default-hertzbeat rule, jsonpath-jsonpath script, website-for website monitoring
-      # 响应数据解析方式: default-系统规则,jsonPath-jsonPath脚本,website-api可用性指标监控
+      authorization:
+        type: ^_^authType^_^
+        basicAuthUsername: ^_^username^_^
+        basicAuthPassword: ^_^password^_^
+        digestAuthUsername: ^_^username^_^
+        digestAuthPassword: ^_^password^_^
       parseType: jsonPath
       parseScript: '$'
diff --git a/manager/src/main/resources/define/app-hertzbeat_token.yml b/manager/src/main/resources/define/app-hertzbeat_token.yml
@@ -185,6 +185,11 @@ metrics:
       timeout: ^_^timeout^_^
       method: GET
       ssl: ^_^ssl^_^
+      authorization:
+        type: Bearer Token
+        # ^o^xxx^o^ ^o^ substitution represents the value of the acquisition metric xxx of the previous priority
+        # ^o^xxx^o^ 替换符表示上一个优先级的采集指标xxx的值
+        bearerTokenToken: ^o^token^o^
       parseType: jsonPath
       parseScript: '$.data'
 
@@ -201,6 +206,11 @@ metrics:
       url: /actuator/metrics/jvm.threads.states
       method: GET
       ssl: ^_^ssl^_^
+      authorization:
+        type: Bearer Token
+        # ^o^xxx^o^ ^o^ substitution represents the value of the acquisition metric xxx of the previous priority
+        # ^o^xxx^o^ 替换符表示上一个优先级的采集指标xxx的值
+        bearerTokenToken: ^o^token^o^
       parseType: jsonPath
       parseScript: '$.availableTags[?(@.tag == "state")].values[*]'
 
@@ -223,6 +233,11 @@ metrics:
       url: /actuator/metrics/jvm.threads.states?tag=state:^o^state^o^
       method: GET
       ssl: ^_^ssl^_^
+      authorization:
+        type: Bearer Token
+        # ^o^xxx^o^ ^o^ substitution represents the value of the acquisition metric xxx of the previous priority
+        # ^o^xxx^o^ 替换符表示上一个优先级的采集指标xxx的值
+        bearerTokenToken: ^o^token^o^
       parseType: jsonPath
       parseScript: '$'
 
@@ -239,6 +254,11 @@ metrics:
       url: /actuator/metrics/jvm.memory.used
       method: GET
       ssl: ^_^ssl^_^
+      authorization:
+        type: Bearer Token
+        # ^o^xxx^o^ ^o^ substitution represents the value of the acquisition metric xxx of the previous priority
+        # ^o^xxx^o^ 替换符表示上一个优先级的采集指标xxx的值
+        bearerTokenToken: ^o^token^o^
       parseType: jsonPath
       parseScript: '$.availableTags[?(@.tag == "id")].values[*]'
 
@@ -264,5 +284,10 @@ metrics:
       url: /actuator/metrics/jvm.memory.used?tag=id:^o^id^o^
       method: GET
       ssl: ^_^ssl^_^
+      authorization:
+        type: Bearer Token
+        # ^o^xxx^o^ ^o^ substitution represents the value of the acquisition metric xxx of the previous priority
+        # ^o^xxx^o^ 替换符表示上一个优先级的采集指标xxx的值
+        bearerTokenToken: ^o^token^o^
       parseType: jsonPath
       parseScript: '$'
diff --git a/manager/src/main/resources/sureness.yml b/manager/src/main/resources/sureness.yml
@@ -55,12 +55,10 @@ resourceRole:
 # 需要被过滤保护的资源,不认证鉴权直接访问
 # /api/v1/source3===get 表示 /api/v1/source3===get 可以被任何人访问 无需登录认证鉴权
 excludedResource:
-  - /api/alerts/**===*
+  - /api/alerts/report/**===*
   - /api/account/auth/**===*
   - /api/i18n/**===get
-  - /api/metrics===get
   - /api/apps/hierarchy===get
-  - /actuator/**===get
   - /api/push/**===*
   # web ui 前端静态资源
   - /===get

diff --git a/script/application.yml b/script/application.yml
@@ -45,7 +45,6 @@ management:
         include:
           - 'metrics'
           - 'health'
-          - 'env'
     enabled-by-default: on
 
 sureness:

diff --git a/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml b/script/docker-compose/hertzbeat-mysql-iotdb/conf/application.yml
@@ -45,7 +45,6 @@ management:
         include:
           - 'metrics'
           - 'health'
-          - 'env'
     enabled-by-default: on
 
 sureness: