Merge pull request #314 from Nizernizer/main

fix : clear invalid byte[] and api fallback.
HXSecurity · May 19, 2022 · 329372f · 329372f
2 parents 3352d46 + 43a71ea
commit 329372f
Show file tree

Hide file tree

Showing 15 changed files with 131 additions and 29 deletions.
diff --git a/dongtai-agent/src/main/java/io/dongtai/iast/agent/Agent.java b/dongtai-agent/src/main/java/io/dongtai/iast/agent/Agent.java
@@ -2,6 +2,7 @@
 
 import java.io.*;
 import java.util.Arrays;
+import java.util.UUID;
 
 import io.dongtai.iast.agent.util.FileUtils;
 import io.dongtai.log.DongTaiLog;
@@ -127,13 +128,13 @@ public static boolean isMacOs() {
 
     private static void extractJattach() throws IOException {
         if (isWindows()) {
-            JATTACH_FILE = System.getProperty("java.io.tmpdir.dongtai") + "iast" + File.separator + "jattach.exe";
+            JATTACH_FILE = System.getProperty("java.io.tmpdir")+ File.separator + "iast"+System.currentTimeMillis() + File.separator + "jattach.exe";
             FileUtils.getResourceToFile("bin/jattach.exe", JATTACH_FILE);
         } else if (isMacOs()) {
-            JATTACH_FILE = System.getProperty("java.io.tmpdir.dongtai") + "iast" + File.separator + "jattach-mac";
+            JATTACH_FILE = System.getProperty("java.io.tmpdir")+ File.separator + "iast"+System.currentTimeMillis() + File.separator + "jattach-mac";
             FileUtils.getResourceToFile("bin/jattach-mac", JATTACH_FILE);
         } else {
-            JATTACH_FILE = System.getProperty("java.io.tmpdir.dongtai") + "iast" + File.separator + "jattach-linux";
+            JATTACH_FILE = System.getProperty("java.io.tmpdir")+ File.separator + "iast"+System.currentTimeMillis() + File.separator + "jattach-linux";
             FileUtils.getResourceToFile("bin/jattach-linux", JATTACH_FILE);
         }
         if ((new File(JATTACH_FILE)).setExecutable(true)) {

diff --git a/dongtai-agent/src/main/java/io/dongtai/iast/agent/manager/EngineManager.java b/dongtai-agent/src/main/java/io/dongtai/iast/agent/manager/EngineManager.java
@@ -218,6 +218,7 @@ private boolean downloadJarPackageToCacheFromUrl(String fileUrl, String fileName
                 while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) {
                     fileOutputStream.write(dataBuffer, 0, bytesRead);
                 }
+                dataBuffer = null;
                 in.close();
                 fileOutputStream.close();
                 DongTaiLog.info("The remote file " + fileUrl + " was successfully written to the local cache.");

diff --git a/dongtai-agent/src/main/java/io/dongtai/iast/agent/monitor/impl/PerformanceMonitor.java b/dongtai-agent/src/main/java/io/dongtai/iast/agent/monitor/impl/PerformanceMonitor.java
@@ -81,7 +81,7 @@ public static List<PerformanceMetrics> getPerformanceMetrics() {
     }
 
     public static Integer checkThresholdValue() {
-/*        int thresholdValue = 100;
+        int thresholdValue = 100;
         try {
             String respRaw = getThresholdValue();
             if (respRaw != null && !respRaw.isEmpty()) {
@@ -91,7 +91,7 @@ public static Integer checkThresholdValue() {
                 thresholdValue = Integer.parseInt(cpuLimit.get("value").toString());
             }
         } catch (Exception ignored) {
-        }*/
+        }
         return 100;
     }
 
@@ -131,10 +131,10 @@ public void check() throws Exception {
         updatePerformanceMetrics(performanceMetrics);
         // 检查性能指标(用于熔断降级)
         checkPerformanceMetrics(performanceMetrics);
-        int UsedRate = CPU_USAGE;
+/*        int UsedRate = CPU_USAGE;
         PerformanceMonitor.AGENT_THRESHOLD_VALUE = PerformanceMonitor.checkThresholdValue();
         int preStatus = this.engineManager.getRunningStatus();
-/*        if (isStart(UsedRate, preStatus)) {
+        if (isStart(UsedRate, preStatus)) {
             this.engineManager.start();
             DongTaiLog.info("The current CPU usage is " + UsedRate + "%, lower than the threshold " + AGENT_THRESHOLD_VALUE + "%，and the detection engine is starting");
         } else if (isStop(UsedRate, preStatus)) {

diff --git a/dongtai-agent/src/main/java/io/dongtai/iast/agent/util/FileUtils.java b/dongtai-agent/src/main/java/io/dongtai/iast/agent/util/FileUtils.java
@@ -30,7 +30,7 @@ public static boolean getResourceToFile(String resourceName, String fileName) th
             while ((length = is.read(data)) != -1) {
                 fos.write(data, 0, length);
             }
-
+            data = null;
             is.close();
             fos.close();
             return true;

diff --git a/dongtai-agent/src/main/java/io/dongtai/iast/agent/util/base64/CharacterDecoder.java b/dongtai-agent/src/main/java/io/dongtai/iast/agent/util/base64/CharacterDecoder.java
@@ -82,6 +82,7 @@ public byte[] decodeBuffer(String var1) throws IOException {
         ByteArrayInputStream var3 = new ByteArrayInputStream(var2);
         ByteArrayOutputStream var4 = new ByteArrayOutputStream();
         this.decodeBuffer(var3, var4);
+        var2 = null;
         return var4.toByteArray();
     }
 

diff --git a/dongtai-agent/src/main/java/io/dongtai/iast/agent/util/base64/CharacterEncoder.java b/dongtai-agent/src/main/java/io/dongtai/iast/agent/util/base64/CharacterEncoder.java
@@ -75,6 +75,7 @@ public void encode(InputStream var1, OutputStream var2) throws IOException {
         }
 
         this.encodeBufferSuffix(var2);
+        var5 = null;
     }
 
     public void encode(byte[] var1, OutputStream var2) throws IOException {
@@ -150,6 +151,7 @@ public void encodeBuffer(InputStream var1, OutputStream var2) throws IOException
         } while (var4 >= this.bytesPerLine());
 
         this.encodeBufferSuffix(var2);
+        var5 = null;
     }
 
     public void encodeBuffer(byte[] var1, OutputStream var2) throws IOException {

diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/IastClassFileTransformer.java b/dongtai-core/src/main/java/io/dongtai/iast/core/bytecode/IastClassFileTransformer.java
@@ -158,10 +158,11 @@ public byte[] transform(final ClassLoader loader,
                         return dumpClassIfNecessary(cr.getClassName(), cw.toByteArray(), srcByteCodeArray);
                     }
                 }
+                sourceCodeBak = null;
             }
         } catch (
-                Throwable ignore) {
-            DongTaiLog.debug(ignore);
+                Throwable throwable) {
+            DongTaiLog.debug(throwable);
         } finally {
             if (isRunning) {
                 EngineManager.turnOnDongTai();

diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/SpyDispatcherImpl.java b/dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/SpyDispatcherImpl.java
@@ -12,6 +12,8 @@
 import io.dongtai.iast.core.handler.hookpoint.service.ServiceHandler;
 import io.dongtai.iast.core.handler.hookpoint.service.kafka.KafkaHandler;
 import io.dongtai.iast.core.service.ErrorLogReport;
+import io.dongtai.iast.core.utils.config.RemoteConfigUtils;
+import io.dongtai.iast.core.utils.matcher.ConfigMatcher;
 import io.dongtai.log.DongTaiLog;
 
 import java.lang.dongtai.SpyDispatcher;
@@ -57,9 +59,15 @@ public void leaveHttp(Object request, Object response) {
                     GraphBuilder.buildAndReport(request, response);
                     EngineManager.cleanThreadState();
                     long responseTimeEnd = System.currentTimeMillis()-responseTime.get()+8;
-                    DongTaiLog.debug(GraphBuilder.getURL() + " response time: "+responseTimeEnd+"ms");
+                    DongTaiLog.debug("url {} response time: {} ms",GraphBuilder.getURL(),responseTimeEnd);
+                    if (RemoteConfigUtils.enableAutoFallback() && responseTimeEnd > RemoteConfigUtils.getApiResponseTime(null)){
+                        RemoteConfigUtils.fallbackReqCount++;
+                        DongTaiLog.warn("url {} response time: {} ms, greater than {} ms",GraphBuilder.getURL(),responseTimeEnd,RemoteConfigUtils.getApiResponseTime(null));
+                        if (!"/".equals(GraphBuilder.getURL())){
+                            ConfigMatcher.getInstance().FALLBACK_URL.add(GraphBuilder.getURI());
+                        }
+                    }
                 }
-
                 EngineManager.turnOnDongTai();
             }
         } catch (Exception e) {

diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/graphy/GraphBuilder.java b/dongtai-core/src/main/java/io/dongtai/iast/core/handler/hookpoint/graphy/GraphBuilder.java
@@ -23,6 +23,7 @@
 public class GraphBuilder {
 
     private static String URL;
+    private static String URI;
 
     public static void buildAndReport(Object request, Object response) {
         List<GraphNode> nodeList = build();
@@ -100,6 +101,7 @@ public static String convertToReport(List<GraphNode> nodeList, Object request, O
         String requestURI = requestMeta.getOrDefault("requestURI", "").toString();
         detail.put(ReportConstant.URI, requestURI);
         setURL(requestURL);
+        setURI(requestURI);
         detail.put(ReportConstant.CLIENT_IP, requestMeta.getOrDefault("remoteAddr", ""));
         detail.put(ReportConstant.QUERY_STRING, requestMeta.getOrDefault("queryString", ""));
         detail.put(ReportConstant.REQ_HEADER,
@@ -144,4 +146,12 @@ public static String getURL() {
     public static void setURL(String URL) {
         GraphBuilder.URL = URL;
     }
+
+    public static String getURI() {
+        return URI;
+    }
+
+    public static void setURI(String URI) {
+        GraphBuilder.URI = URI;
+    }
 }
diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/service/AgentQueueReport.java b/dongtai-core/src/main/java/io/dongtai/iast/core/service/AgentQueueReport.java
@@ -35,13 +35,15 @@ public static String generateHeartBeatMsg() {
 
     @Override
     public void run() {
-        try {
-            StringBuilder replayRequestRaw = HttpClientUtils.sendPost(Constants.API_REPORT_UPLOAD, generateHeartBeatMsg());
-            if (EngineManager.isEngineRunning()){
-                ThreadPools.submitReplayTask(replayRequestRaw);
+        if (EngineManager.isEngineRunning()){
+            try {
+                StringBuilder replayRequestRaw = HttpClientUtils.sendPost(Constants.API_REPORT_UPLOAD, generateHeartBeatMsg());
+                if (EngineManager.isEngineRunning()){
+                    ThreadPools.submitReplayTask(replayRequestRaw);
+                }
+            } catch (Exception e) {
+                DongTaiLog.debug("send API Queue to {} error, reason: {}", Constants.API_REPORT_UPLOAD, e);
             }
-        } catch (Exception e) {
-            DongTaiLog.debug("send API Queue to {} error, reason: {}", Constants.API_REPORT_UPLOAD, e);
         }
     }
 }
diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/utils/Constants.java b/dongtai-core/src/main/java/io/dongtai/iast/core/utils/Constants.java
@@ -4,7 +4,7 @@
  * @author dongzhiyong@huoxian.cn
  */
 public class Constants {
-    public static final String AGENT_VERSION_VALUE = "v1.7.0.test.3";
+    public static final String AGENT_VERSION_VALUE = "v1.7.0";
     public final static String API_REPORT_UPLOAD = "/api/v1/report/upload";
     public final static String SERVER_ADDRESS = "/api/v1/agent/update";
     public final static String API_HOOK_PROFILE = "/api/v1/profiles";

diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/utils/HttpClientUtils.java b/dongtai-core/src/main/java/io/dongtai/iast/core/utils/HttpClientUtils.java
@@ -1,5 +1,6 @@
 package io.dongtai.iast.core.utils;
 
+import io.dongtai.iast.core.EngineManager;
 import io.dongtai.log.DongTaiLog;
 
 import javax.net.ssl.*;
@@ -26,6 +27,7 @@ public class HttpClientUtils {
     public final static HostnameVerifier DO_NOT_VERIFY = new HttpClientHostnameVerifier();
     private final static PropertyUtils PROPERTIES = PropertyUtils.getInstance();
     private final static Proxy PROXY = loadProxy();
+    private static Integer FAILED_CONNECTION_COUNT = 0;
 
     public static StringBuilder sendGet(String uri, String arg, String value) {
         try {
@@ -104,6 +106,12 @@ private static StringBuilder sendRequest(HttpMethods method, String baseUrl, Str
             return response;
         } catch (Exception e) {
             DongTaiLog.error("io.dongtai.iast.core.utils.HttpClientUtils.sendRequest(io.dongtai.iast.core.utils.HttpMethods,java.lang.String,java.lang.String,java.lang.String,java.util.HashMap<java.lang.String,java.lang.String>,java.net.Proxy)",e);
+            FAILED_CONNECTION_COUNT++;
+            if (FAILED_CONNECTION_COUNT > 10){
+                DongTaiLog.error("The network connection is abnormal, DongTai engine is shut down.");
+                EngineManager.turnOffEngine();
+                FAILED_CONNECTION_COUNT = 0;
+            }
         } finally {
             if (connection != null) {
                 connection.disconnect();
@@ -142,6 +150,7 @@ public static void downloadRemoteJar(String fileURI, String fileName) {
             while ((bytesRead = in.read(dataBuffer, 0, 1024)) != -1) {
                 fileOutputStream.write(dataBuffer, 0, bytesRead);
             }
+            dataBuffer = null;
             in.close();
             fileOutputStream.close();
             DongTaiLog.info("The remote file {} was successfully written to the local cache", fileURI);

diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/utils/base64/CharacterEncoder.java b/dongtai-core/src/main/java/io/dongtai/iast/core/utils/base64/CharacterEncoder.java
@@ -152,6 +152,7 @@ public void encodeBuffer(InputStream var1, OutputStream var2) throws IOException
         } while (var4 >= this.bytesPerLine());
 
         this.encodeBufferSuffix(var2);
+        var5 = null;
     }
 
     public void encodeBuffer(byte[] var1, OutputStream var2) throws IOException {

diff --git a/dongtai-core/src/main/java/io/dongtai/iast/core/utils/config/RemoteConfigUtils.java b/dongtai-core/src/main/java/io/dongtai/iast/core/utils/config/RemoteConfigUtils.java
@@ -17,6 +17,7 @@
 import io.dongtai.iast.core.utils.config.entity.PerformanceLimitThreshold;
 import io.dongtai.iast.core.utils.config.entity.RemoteConfigEntity;
 import io.dongtai.iast.core.utils.json.GsonUtils;
+import io.dongtai.iast.core.utils.matcher.ConfigMatcher;
 import io.dongtai.log.DongTaiLog;
 import org.json.JSONObject;
 
@@ -70,6 +71,15 @@ private RemoteConfigUtils() {
     private static Double secondFallbackFrequencyInitBurstSeconds;
     private static Long secondFallbackDuration;
 
+    /**
+     * 接口响应时间相关
+     */
+    private static Double apiResponseTime;
+    public static Integer fallbackReqCount = 0;
+    private static Integer reqCount = 0;
+    private static Integer lastReqCount = 0;
+    private static Double responseTimeThresholdRate;
+
     private static Boolean systemIsUninstall;
     private static Boolean jvmIsUninstall;
     private static Boolean applicationIsUninstall;
@@ -137,7 +147,7 @@ public static void syncRemoteConfig(int agentId) {
     }
 
     /**
-     * 同步远程配置
+     * 同步远程配置-v2
      *
      * @param agentId agent的唯一标识
      */
@@ -191,6 +201,9 @@ public static void syncRemoteConfigV2(int agentId) {
                             case "heavyTrafficLimitTokenPerSecond":
                                 heavyTrafficLimitTokenPerSecond = performanceEntity.getValue();
                                 break;
+                            case "apiResponseTime":
+                                apiResponseTime = performanceEntity.getValue();
+                                break;
                         }
                     }
                 }
@@ -253,6 +266,19 @@ public static void syncRemoteConfigV2(int agentId) {
                 existsRemoteConfigMeta = remoteResponse;
                 DongTaiLog.debug("Sync remote config successful.");
             }
+            if (enableAutoFallback == true){
+                reqCount = EngineManager.getRequestCount() - lastReqCount;
+                lastReqCount = EngineManager.getRequestCount();
+                if (reqCount>0 && fallbackReqCount*1.0/reqCount > getResponseTimeThresholdRate()){
+                    ConfigMatcher.getInstance().FALLBACK_URL.clear();
+                    EngineManager.turnOffEngine();
+                    DongTaiLog.warn("The response time exceed the threshold, Dongtai engine shut down successfully.");
+                }else if (null != systemIsUninstall && systemIsUninstall && reqCount>0 && EngineManager.enableDongTai==0){
+                    EngineManager.turnOnEngine();
+                    DongTaiLog.info("The response time is below the threshold, Dongtai engine open successfully.");
+                }
+                fallbackReqCount = 0;
+            }
         } catch (Throwable t) {
             DongTaiLog.warn("Sync remote config failed, msg: {}, error: {}", t.getMessage(), t.getCause());
         }
@@ -278,6 +304,9 @@ private static String getConfigFromRemote(int agentId) {
      * 根据agentID获取服务端对Agent的配置
      */
     private static String getConfigFromRemoteV2(int agentId) {
+        if (EngineManager.isEngineRunning()){
+            return "{}";
+        }
         JSONObject report = new JSONObject();
         report.put(KEY_AGENT_ID, agentId);
         try {
@@ -343,12 +372,12 @@ private static RemoteConfigEntityV2 parseRemoteConfigResponseV2(String remoteRes
             if (result.isSuccess()) {
                 return result.getData();
             } else {
-                DongTaiLog.warn("remoteConfig request not success, status:{}, msg:{},response:{}", result.getStatus(), result.getMsg(),
+                DongTaiLog.debug("remoteConfig request not success, status:{}, msg:{},response:{}", result.getStatus(), result.getMsg(),
                         GsonUtils.toJson(remoteResponse));
                 return null;
             }
         } catch (Throwable t) {
-            DongTaiLog.warn("remoteConfig parse failed: msg:{}, err:{}, response:{}", t.getMessage(), t.getCause(), GsonUtils.toJson(remoteResponse));
+            DongTaiLog.debug("remoteConfig parse failed: msg:{}, err:{}, response:{}", t.getMessage(), t.getCause(), GsonUtils.toJson(remoteResponse));
             return null;
         }
     }
@@ -596,6 +625,26 @@ public static long getSwitchOpenStatusDurationThreshold(Properties cfg) {
         return secondFallbackDuration;
     }
 
+    /**
+     * 响应时间阈值-降级开关持续时间限制-降级开关打开状态持续最大时间(ms)
+     */
+    public static Double getApiResponseTime(Properties cfg) {
+        if (apiResponseTime == null) {
+            apiResponseTime = Double.valueOf(System.getProperty("dongtai.fallback.response.time", String.valueOf(30000.0)));
+        }
+        return apiResponseTime;
+    }
+
+    /**
+     * 响应时间阈值-降级开关持续时间限制-降级开关打开状态持续最大时间(ms)
+     */
+    public static Double getResponseTimeThresholdRate() {
+        if (responseTimeThresholdRate == null) {
+            responseTimeThresholdRate = Double.valueOf(System.getProperty("dongtai.fallback.response.rate", String.valueOf(0.9)));
+        }
+        return responseTimeThresholdRate;
+    }
+
     /**
      * 从配置文件中构建性能指标
      *
@@ -645,4 +694,20 @@ public static Boolean getApplicationIsUninstall() {
         }
         return applicationIsUninstall;
     }
+
+    public static Integer getFallbackReqCount() {
+        return fallbackReqCount;
+    }
+
+    public static void setFallbackReqCount(Integer fallbackReqCount) {
+        RemoteConfigUtils.fallbackReqCount = fallbackReqCount;
+    }
+
+    public static Integer getReqCount() {
+        return reqCount;
+    }
+
+    public static void setReqCount(Integer reqCount) {
+        RemoteConfigUtils.reqCount = reqCount;
+    }
 }